PostgreSQL Source Code git master
wparser_def.c
Go to the documentation of this file.
1/*-------------------------------------------------------------------------
2 *
3 * wparser_def.c
4 * Default text search parser
5 *
6 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
7 *
8 *
9 * IDENTIFICATION
10 * src/backend/tsearch/wparser_def.c
11 *
12 *-------------------------------------------------------------------------
13 */
14
15#include "postgres.h"
16
17#include <limits.h>
18#include <wctype.h>
19
20#include "commands/defrem.h"
21#include "mb/pg_wchar.h"
22#include "miscadmin.h"
23#include "tsearch/ts_public.h"
24#include "tsearch/ts_type.h"
25#include "tsearch/ts_utils.h"
26#include "utils/builtins.h"
27#include "utils/pg_locale.h"
28
29
30/* Define me to enable tracing of parser behavior */
31/* #define WPARSER_TRACE */
32
33
34/* Output token categories */
35
36#define ASCIIWORD 1
37#define WORD_T 2
38#define NUMWORD 3
39#define EMAIL 4
40#define URL_T 5
41#define HOST 6
42#define SCIENTIFIC 7
43#define VERSIONNUMBER 8
44#define NUMPARTHWORD 9
45#define PARTHWORD 10
46#define ASCIIPARTHWORD 11
47#define SPACE 12
48#define TAG_T 13
49#define PROTOCOL 14
50#define NUMHWORD 15
51#define ASCIIHWORD 16
52#define HWORD 17
53#define URLPATH 18
54#define FILEPATH 19
55#define DECIMAL_T 20
56#define SIGNEDINT 21
57#define UNSIGNEDINT 22
58#define XMLENTITY 23
59
60#define LASTNUM 23
61
62static const char *const tok_alias[] = {
63 "",
64 "asciiword",
65 "word",
66 "numword",
67 "email",
68 "url",
69 "host",
70 "sfloat",
71 "version",
72 "hword_numpart",
73 "hword_part",
74 "hword_asciipart",
75 "blank",
76 "tag",
77 "protocol",
78 "numhword",
79 "asciihword",
80 "hword",
81 "url_path",
82 "file",
83 "float",
84 "int",
85 "uint",
86 "entity"
87};
88
89static const char *const lex_descr[] = {
90 "",
91 "Word, all ASCII",
92 "Word, all letters",
93 "Word, letters and digits",
94 "Email address",
95 "URL",
96 "Host",
97 "Scientific notation",
98 "Version number",
99 "Hyphenated word part, letters and digits",
100 "Hyphenated word part, all letters",
101 "Hyphenated word part, all ASCII",
102 "Space symbols",
103 "XML tag",
104 "Protocol head",
105 "Hyphenated word, letters and digits",
106 "Hyphenated word, all ASCII",
107 "Hyphenated word, all letters",
108 "URL path",
109 "File or path name",
110 "Decimal notation",
111 "Signed integer",
112 "Unsigned integer",
113 "XML entity"
114};
115
116
117/* Parser states */
118
119typedef enum
120{
121 TPS_Base = 0,
122 TPS_InNumWord,
123 TPS_InAsciiWord,
124 TPS_InWord,
125 TPS_InUnsignedInt,
126 TPS_InSignedIntFirst,
127 TPS_InSignedInt,
128 TPS_InSpace,
129 TPS_InUDecimalFirst,
130 TPS_InUDecimal,
131 TPS_InDecimalFirst,
132 TPS_InDecimal,
133 TPS_InVerVersion,
134 TPS_InSVerVersion,
135 TPS_InVersionFirst,
136 TPS_InVersion,
137 TPS_InMantissaFirst,
138 TPS_InMantissaSign,
139 TPS_InMantissa,
140 TPS_InXMLEntityFirst,
141 TPS_InXMLEntity,
142 TPS_InXMLEntityNumFirst,
143 TPS_InXMLEntityNum,
144 TPS_InXMLEntityHexNumFirst,
145 TPS_InXMLEntityHexNum,
146 TPS_InXMLEntityEnd,
147 TPS_InTagFirst,
148 TPS_InXMLBegin,
149 TPS_InTagCloseFirst,
150 TPS_InTagName,
151 TPS_InTagBeginEnd,
152 TPS_InTag,
153 TPS_InTagEscapeK,
154 TPS_InTagEscapeKK,
155 TPS_InTagBackSleshed,
156 TPS_InTagEnd,
157 TPS_InCommentFirst,
158 TPS_InCommentLast,
159 TPS_InComment,
160 TPS_InCloseCommentFirst,
161 TPS_InCloseCommentLast,
162 TPS_InCommentEnd,
163 TPS_InHostFirstDomain,
164 TPS_InHostDomainSecond,
165 TPS_InHostDomain,
166 TPS_InPortFirst,
167 TPS_InPort,
168 TPS_InHostFirstAN,
169 TPS_InHost,
170 TPS_InEmail,
171 TPS_InFileFirst,
172 TPS_InFileTwiddle,
173 TPS_InPathFirst,
174 TPS_InPathFirstFirst,
175 TPS_InPathSecond,
176 TPS_InFile,
177 TPS_InFileNext,
178 TPS_InURLPathFirst,
179 TPS_InURLPathStart,
180 TPS_InURLPath,
181 TPS_InFURL,
182 TPS_InProtocolFirst,
183 TPS_InProtocolSecond,
184 TPS_InProtocolEnd,
185 TPS_InHyphenAsciiWordFirst,
186 TPS_InHyphenAsciiWord,
187 TPS_InHyphenWordFirst,
188 TPS_InHyphenWord,
189 TPS_InHyphenNumWordFirst,
190 TPS_InHyphenNumWord,
191 TPS_InHyphenDigitLookahead,
192 TPS_InParseHyphen,
193 TPS_InParseHyphenHyphen,
194 TPS_InHyphenWordPart,
195 TPS_InHyphenAsciiWordPart,
196 TPS_InHyphenNumWordPart,
197 TPS_InHyphenUnsignedInt,
198 TPS_Null /* last state (fake value) */
199} TParserState;
200
201/* forward declaration */
202struct TParser;
203
204typedef int (*TParserCharTest) (struct TParser *); /* any p_is* functions
205 * except p_iseq */
206typedef void (*TParserSpecial) (struct TParser *); /* special handler for
207 * special cases... */
208
209typedef struct
210{
211 TParserCharTest isclass;
212 char c;
213 uint16 flags;
214 TParserState tostate;
215 int type;
216 TParserSpecial special;
217} TParserStateActionItem;
218
219/* Flag bits in TParserStateActionItem.flags */
220#define A_NEXT 0x0000
221#define A_BINGO 0x0001
222#define A_POP 0x0002
223#define A_PUSH 0x0004
224#define A_RERUN 0x0008
225#define A_CLEAR 0x0010
226#define A_MERGE 0x0020
227#define A_CLRALL 0x0040
228
229typedef struct TParserPosition
230{
231 int posbyte; /* position of parser in bytes */
232 int poschar; /* position of parser in characters */
233 int charlen; /* length of current char */
234 int lenbytetoken; /* length of token-so-far in bytes */
235 int lenchartoken; /* and in chars */
236 TParserState state;
237 struct TParserPosition *prev;
238 const TParserStateActionItem *pushedAtAction;
239} TParserPosition;
240
241typedef struct TParser
242{
243 /* string and position information */
244 char *str; /* multibyte string */
245 int lenstr; /* length of mbstring */
246 pg_wchar *pgwstr; /* wide character string for C-locale */
247
248 /* State of parse */
249 int charmaxlen;
250 TParserPosition *state;
251 bool ignore;
252 bool wanthost;
253
254 /* silly char */
255 char c;
256
257 /* out */
258 char *token;
259 int lenbytetoken;
260 int lenchartoken;
261 int type;
262} TParser;
263
264
265/* forward decls here */
266static bool TParserGet(TParser *prs);
267
268
269static TParserPosition *
270newTParserPosition(TParserPosition *prev)
271{
272 TParserPosition *res = (TParserPosition *) palloc(sizeof(TParserPosition));
273
274 if (prev)
275 memcpy(res, prev, sizeof(TParserPosition));
276 else
277 memset(res, 0, sizeof(TParserPosition));
278
279 res->prev = prev;
280
281 res->pushedAtAction = NULL;
282
283 return res;
284}
285
286static TParser *
287TParserInit(char *str, int len)
288{
289 TParser *prs = (TParser *) palloc0(sizeof(TParser));
290
291 prs->charmaxlen = pg_database_encoding_max_length();
292 prs->str = str;
293 prs->lenstr = len;
294 prs->pgwstr = (pg_wchar *) palloc(sizeof(pg_wchar) * (prs->lenstr + 1));
295 pg_mb2wchar_with_len(prs->str, prs->pgwstr, prs->lenstr);
296
297 prs->state = newTParserPosition(NULL);
298 prs->state->state = TPS_Base;
299
300#ifdef WPARSER_TRACE
301 fprintf(stderr, "parsing \"%.*s\"\n", len, str);
302#endif
303
304 return prs;
305}
306
307/*
308 * As an alternative to a full TParserInit one can create a
309 * TParserCopy which basically is a regular TParser without a private
310 * copy of the string - instead it uses the one from another TParser.
311 * This is useful because at some places TParsers are created
312 * recursively and the repeated copying around of the strings can
313 * cause major inefficiency if the source string is long.
314 * The new parser starts parsing at the original's current position.
315 *
316 * Obviously one must not close the original TParser before the copy.
317 */
318static TParser *
319TParserCopyInit(const TParser *orig)
320{
321 TParser *prs = (TParser *) palloc0(sizeof(TParser));
322
323 prs->charmaxlen = orig->charmaxlen;
324 prs->str = orig->str + orig->state->posbyte;
325 prs->lenstr = orig->lenstr - orig->state->posbyte;
326
327 if (orig->pgwstr)
328 prs->pgwstr = orig->pgwstr + orig->state->poschar;
329
330 prs->state = newTParserPosition(NULL);
331 prs->state->state = TPS_Base;
332
333#ifdef WPARSER_TRACE
334 fprintf(stderr, "parsing copy of \"%.*s\"\n", prs->lenstr, prs->str);
335#endif
336
337 return prs;
338}
339
340
341static void
342TParserClose(TParser *prs)
343{
344 while (prs->state)
345 {
346 TParserPosition *ptr = prs->state->prev;
347
348 pfree(prs->state);
349 prs->state = ptr;
350 }
351
352 if (prs->pgwstr)
353 pfree(prs->pgwstr);
354
355#ifdef WPARSER_TRACE
356 fprintf(stderr, "closing parser\n");
357#endif
358 pfree(prs);
359}
360
361/*
362 * Close a parser created with TParserCopyInit
363 */
364static void
365TParserCopyClose(TParser *prs)
366{
367 while (prs->state)
368 {
369 TParserPosition *ptr = prs->state->prev;
370
371 pfree(prs->state);
372 prs->state = ptr;
373 }
374
375#ifdef WPARSER_TRACE
376 fprintf(stderr, "closing parser copy\n");
377#endif
378 pfree(prs);
379}
380
381
382/*
383 * Character-type support functions using the database default locale. If the
384 * locale is C, and the input character is non-ascii, the value to be returned
385 * is determined by the 'nonascii' macro argument.
386 */
387
388#define p_iswhat(type, nonascii) \
389 \
390static int \
391p_is##type(TParser *prs) \
392{ \
393 pg_locale_t locale = pg_database_locale(); \
394 pg_wchar wc; \
395 Assert(prs->state); \
396 wc = prs->pgwstr[prs->state->poschar]; \
397 if (prs->charmaxlen > 1 && locale->ctype_is_c && wc > 0x7f) \
398 return nonascii; \
399 return pg_isw##type(wc, pg_database_locale()); \
400} \
401 \
402static int \
403p_isnot##type(TParser *prs) \
404{ \
405 return !p_is##type(prs); \
406}
407
408/*
409 * In C locale with a multibyte encoding, any non-ASCII symbol is considered
410 * an alpha character, but not a member of other char classes.
411 */
412p_iswhat(alnum, 1)
413p_iswhat(alpha, 1)
414p_iswhat(digit, 0)
415p_iswhat(lower, 0)
416p_iswhat(print, 0)
417p_iswhat(punct, 0)
418p_iswhat(space, 0)
419p_iswhat(upper, 0)
420p_iswhat(xdigit, 0)
421
422/* p_iseq should be used only for ascii symbols */
423
424static int
425p_iseq(TParser *prs, char c)
426{
427 Assert(prs->state);
428 return ((prs->state->charlen == 1 && *(prs->str + prs->state->posbyte) == c)) ? 1 : 0;
429}
430
431static int
432p_isEOF(TParser *prs)
433{
434 Assert(prs->state);
435 return (prs->state->posbyte == prs->lenstr || prs->state->charlen == 0) ? 1 : 0;
436}
437
438static int
439p_iseqC(TParser *prs)
440{
441 return p_iseq(prs, prs->c);
442}
443
444static int
445p_isneC(TParser *prs)
446{
447 return !p_iseq(prs, prs->c);
448}
449
450static int
451p_isascii(TParser *prs)
452{
453 return (prs->state->charlen == 1 && isascii((unsigned char) *(prs->str + prs->state->posbyte))) ? 1 : 0;
454}
455
456static int
457p_isasclet(TParser *prs)
458{
459 return (p_isascii(prs) && p_isalpha(prs)) ? 1 : 0;
460}
461
462static int
463p_isurlchar(TParser *prs)
464{
465 char ch;
466
467 /* no non-ASCII need apply */
468 if (prs->state->charlen != 1)
469 return 0;
470 ch = *(prs->str + prs->state->posbyte);
471 /* no spaces or control characters */
472 if (ch <= 0x20 || ch >= 0x7F)
473 return 0;
474 /* reject characters disallowed by RFC 3986 */
475 switch (ch)
476 {
477 case '"':
478 case '<':
479 case '>':
480 case '\\':
481 case '^':
482 case '`':
483 case '{':
484 case '|':
485 case '}':
486 return 0;
487 }
488 return 1;
489}
490
491
492/* deliberately suppress unused-function complaints for the above */
493void _make_compiler_happy(void);
494void
495_make_compiler_happy(void)
496{
497 p_isalnum(NULL);
498 p_isnotalnum(NULL);
499 p_isalpha(NULL);
500 p_isnotalpha(NULL);
501 p_isdigit(NULL);
502 p_isnotdigit(NULL);
503 p_islower(NULL);
504 p_isnotlower(NULL);
505 p_isprint(NULL);
506 p_isnotprint(NULL);
507 p_ispunct(NULL);
508 p_isnotpunct(NULL);
509 p_isspace(NULL);
510 p_isnotspace(NULL);
511 p_isupper(NULL);
512 p_isnotupper(NULL);
513 p_isxdigit(NULL);
514 p_isnotxdigit(NULL);
515 p_isEOF(NULL);
516 p_iseqC(NULL);
517 p_isneC(NULL);
518}
519
520
521static void
522SpecialTags(TParser *prs)
523{
524 switch (prs->state->lenchartoken)
525 {
526 case 8: /* </script */
527 if (pg_strncasecmp(prs->token, "</script", 8) == 0)
528 prs->ignore = false;
529 break;
530 case 7: /* <script || </style */
531 if (pg_strncasecmp(prs->token, "</style", 7) == 0)
532 prs->ignore = false;
533 else if (pg_strncasecmp(prs->token, "<script", 7) == 0)
534 prs->ignore = true;
535 break;
536 case 6: /* <style */
537 if (pg_strncasecmp(prs->token, "<style", 6) == 0)
538 prs->ignore = true;
539 break;
540 default:
541 break;
542 }
543}
544
545static void
546SpecialFURL(TParser *prs)
547{
548 prs->wanthost = true;
549 prs->state->posbyte -= prs->state->lenbytetoken;
550 prs->state->poschar -= prs->state->lenchartoken;
551}
552
553static void
554SpecialHyphen(TParser *prs)
555{
556 prs->state->posbyte -= prs->state->lenbytetoken;
557 prs->state->poschar -= prs->state->lenchartoken;
558}
559
560static void
561SpecialVerVersion(TParser *prs)
562{
563 prs->state->posbyte -= prs->state->lenbytetoken;
564 prs->state->poschar -= prs->state->lenchartoken;
565 prs->state->lenbytetoken = 0;
566 prs->state->lenchartoken = 0;
567}
568
569static int
570p_isstophost(TParser *prs)
571{
572 if (prs->wanthost)
573 {
574 prs->wanthost = false;
575 return 1;
576 }
577 return 0;
578}
579
580static int
581p_isignore(TParser *prs)
582{
583 return (prs->ignore) ? 1 : 0;
584}
585
586static int
587p_ishost(TParser *prs)
588{
589 TParser *tmpprs = TParserCopyInit(prs);
590 int res = 0;
591
592 tmpprs->wanthost = true;
593
594 /*
595 * Check stack depth before recursing. (Since TParserGet() doesn't
596 * normally recurse, we put the cost of checking here not there.)
597 */
598 check_stack_depth();
599
600 if (TParserGet(tmpprs) && tmpprs->type == HOST)
601 {
602 prs->state->posbyte += tmpprs->lenbytetoken;
603 prs->state->poschar += tmpprs->lenchartoken;
604 prs->state->lenbytetoken += tmpprs->lenbytetoken;
605 prs->state->lenchartoken += tmpprs->lenchartoken;
606 prs->state->charlen = tmpprs->state->charlen;
607 res = 1;
608 }
609 TParserCopyClose(tmpprs);
610
611 return res;
612}
613
614static int
615p_isURLPath(TParser *prs)
616{
617 TParser *tmpprs = TParserCopyInit(prs);
618 int res = 0;
619
620 tmpprs->state = newTParserPosition(tmpprs->state);
621 tmpprs->state->state = TPS_InURLPathFirst;
622
623 /*
624 * Check stack depth before recursing. (Since TParserGet() doesn't
625 * normally recurse, we put the cost of checking here not there.)
626 */
627 check_stack_depth();
628
629 if (TParserGet(tmpprs) && tmpprs->type == URLPATH)
630 {
631 prs->state->posbyte += tmpprs->lenbytetoken;
632 prs->state->poschar += tmpprs->lenchartoken;
633 prs->state->lenbytetoken += tmpprs->lenbytetoken;
634 prs->state->lenchartoken += tmpprs->lenchartoken;
635 prs->state->charlen = tmpprs->state->charlen;
636 res = 1;
637 }
638 TParserCopyClose(tmpprs);
639
640 return res;
641}
642
643/*
644 * returns true if current character has zero display length or
645 * it's a special sign in several languages. Such characters
646 * aren't a word-breaker although they aren't an isalpha.
647 * In beginning of word they aren't a part of it.
648 */
649static int
650p_isspecial(TParser *prs)
651{
652 /*
653 * pg_dsplen could return -1 which means error or control character
654 */
655 if (pg_dsplen(prs->str + prs->state->posbyte) == 0)
656 return 1;
657
658 /*
659 * Unicode Characters in the 'Mark, Spacing Combining' Category That
660 * characters are not alpha although they are not breakers of word too.
661 * Check that only in utf encoding, because other encodings aren't
662 * supported by postgres or even exists.
663 */
664 if (GetDatabaseEncoding() == PG_UTF8)
665 {
666 static const pg_wchar strange_letter[] = {
667 /*
668 * use binary search, so elements should be ordered
669 */
670 0x0903, /* DEVANAGARI SIGN VISARGA */
671 0x093E, /* DEVANAGARI VOWEL SIGN AA */
672 0x093F, /* DEVANAGARI VOWEL SIGN I */
673 0x0940, /* DEVANAGARI VOWEL SIGN II */
674 0x0949, /* DEVANAGARI VOWEL SIGN CANDRA O */
675 0x094A, /* DEVANAGARI VOWEL SIGN SHORT O */
676 0x094B, /* DEVANAGARI VOWEL SIGN O */
677 0x094C, /* DEVANAGARI VOWEL SIGN AU */
678 0x0982, /* BENGALI SIGN ANUSVARA */
679 0x0983, /* BENGALI SIGN VISARGA */
680 0x09BE, /* BENGALI VOWEL SIGN AA */
681 0x09BF, /* BENGALI VOWEL SIGN I */
682 0x09C0, /* BENGALI VOWEL SIGN II */
683 0x09C7, /* BENGALI VOWEL SIGN E */
684 0x09C8, /* BENGALI VOWEL SIGN AI */
685 0x09CB, /* BENGALI VOWEL SIGN O */
686 0x09CC, /* BENGALI VOWEL SIGN AU */
687 0x09D7, /* BENGALI AU LENGTH MARK */
688 0x0A03, /* GURMUKHI SIGN VISARGA */
689 0x0A3E, /* GURMUKHI VOWEL SIGN AA */
690 0x0A3F, /* GURMUKHI VOWEL SIGN I */
691 0x0A40, /* GURMUKHI VOWEL SIGN II */
692 0x0A83, /* GUJARATI SIGN VISARGA */
693 0x0ABE, /* GUJARATI VOWEL SIGN AA */
694 0x0ABF, /* GUJARATI VOWEL SIGN I */
695 0x0AC0, /* GUJARATI VOWEL SIGN II */
696 0x0AC9, /* GUJARATI VOWEL SIGN CANDRA O */
697 0x0ACB, /* GUJARATI VOWEL SIGN O */
698 0x0ACC, /* GUJARATI VOWEL SIGN AU */
699 0x0B02, /* ORIYA SIGN ANUSVARA */
700 0x0B03, /* ORIYA SIGN VISARGA */
701 0x0B3E, /* ORIYA VOWEL SIGN AA */
702 0x0B40, /* ORIYA VOWEL SIGN II */
703 0x0B47, /* ORIYA VOWEL SIGN E */
704 0x0B48, /* ORIYA VOWEL SIGN AI */
705 0x0B4B, /* ORIYA VOWEL SIGN O */
706 0x0B4C, /* ORIYA VOWEL SIGN AU */
707 0x0B57, /* ORIYA AU LENGTH MARK */
708 0x0BBE, /* TAMIL VOWEL SIGN AA */
709 0x0BBF, /* TAMIL VOWEL SIGN I */
710 0x0BC1, /* TAMIL VOWEL SIGN U */
711 0x0BC2, /* TAMIL VOWEL SIGN UU */
712 0x0BC6, /* TAMIL VOWEL SIGN E */
713 0x0BC7, /* TAMIL VOWEL SIGN EE */
714 0x0BC8, /* TAMIL VOWEL SIGN AI */
715 0x0BCA, /* TAMIL VOWEL SIGN O */
716 0x0BCB, /* TAMIL VOWEL SIGN OO */
717 0x0BCC, /* TAMIL VOWEL SIGN AU */
718 0x0BD7, /* TAMIL AU LENGTH MARK */
719 0x0C01, /* TELUGU SIGN CANDRABINDU */
720 0x0C02, /* TELUGU SIGN ANUSVARA */
721 0x0C03, /* TELUGU SIGN VISARGA */
722 0x0C41, /* TELUGU VOWEL SIGN U */
723 0x0C42, /* TELUGU VOWEL SIGN UU */
724 0x0C43, /* TELUGU VOWEL SIGN VOCALIC R */
725 0x0C44, /* TELUGU VOWEL SIGN VOCALIC RR */
726 0x0C82, /* KANNADA SIGN ANUSVARA */
727 0x0C83, /* KANNADA SIGN VISARGA */
728 0x0CBE, /* KANNADA VOWEL SIGN AA */
729 0x0CC0, /* KANNADA VOWEL SIGN II */
730 0x0CC1, /* KANNADA VOWEL SIGN U */
731 0x0CC2, /* KANNADA VOWEL SIGN UU */
732 0x0CC3, /* KANNADA VOWEL SIGN VOCALIC R */
733 0x0CC4, /* KANNADA VOWEL SIGN VOCALIC RR */
734 0x0CC7, /* KANNADA VOWEL SIGN EE */
735 0x0CC8, /* KANNADA VOWEL SIGN AI */
736 0x0CCA, /* KANNADA VOWEL SIGN O */
737 0x0CCB, /* KANNADA VOWEL SIGN OO */
738 0x0CD5, /* KANNADA LENGTH MARK */
739 0x0CD6, /* KANNADA AI LENGTH MARK */
740 0x0D02, /* MALAYALAM SIGN ANUSVARA */
741 0x0D03, /* MALAYALAM SIGN VISARGA */
742 0x0D3E, /* MALAYALAM VOWEL SIGN AA */
743 0x0D3F, /* MALAYALAM VOWEL SIGN I */
744 0x0D40, /* MALAYALAM VOWEL SIGN II */
745 0x0D46, /* MALAYALAM VOWEL SIGN E */
746 0x0D47, /* MALAYALAM VOWEL SIGN EE */
747 0x0D48, /* MALAYALAM VOWEL SIGN AI */
748 0x0D4A, /* MALAYALAM VOWEL SIGN O */
749 0x0D4B, /* MALAYALAM VOWEL SIGN OO */
750 0x0D4C, /* MALAYALAM VOWEL SIGN AU */
751 0x0D57, /* MALAYALAM AU LENGTH MARK */
752 0x0D82, /* SINHALA SIGN ANUSVARAYA */
753 0x0D83, /* SINHALA SIGN VISARGAYA */
754 0x0DCF, /* SINHALA VOWEL SIGN AELA-PILLA */
755 0x0DD0, /* SINHALA VOWEL SIGN KETTI AEDA-PILLA */
756 0x0DD1, /* SINHALA VOWEL SIGN DIGA AEDA-PILLA */
757 0x0DD8, /* SINHALA VOWEL SIGN GAETTA-PILLA */
758 0x0DD9, /* SINHALA VOWEL SIGN KOMBUVA */
759 0x0DDA, /* SINHALA VOWEL SIGN DIGA KOMBUVA */
760 0x0DDB, /* SINHALA VOWEL SIGN KOMBU DEKA */
761 0x0DDC, /* SINHALA VOWEL SIGN KOMBUVA HAA AELA-PILLA */
762 0x0DDD, /* SINHALA VOWEL SIGN KOMBUVA HAA DIGA
763 * AELA-PILLA */
764 0x0DDE, /* SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA */
765 0x0DDF, /* SINHALA VOWEL SIGN GAYANUKITTA */
766 0x0DF2, /* SINHALA VOWEL SIGN DIGA GAETTA-PILLA */
767 0x0DF3, /* SINHALA VOWEL SIGN DIGA GAYANUKITTA */
768 0x0F3E, /* TIBETAN SIGN YAR TSHES */
769 0x0F3F, /* TIBETAN SIGN MAR TSHES */
770 0x0F7F, /* TIBETAN SIGN RNAM BCAD */
771 0x102B, /* MYANMAR VOWEL SIGN TALL AA */
772 0x102C, /* MYANMAR VOWEL SIGN AA */
773 0x1031, /* MYANMAR VOWEL SIGN E */
774 0x1038, /* MYANMAR SIGN VISARGA */
775 0x103B, /* MYANMAR CONSONANT SIGN MEDIAL YA */
776 0x103C, /* MYANMAR CONSONANT SIGN MEDIAL RA */
777 0x1056, /* MYANMAR VOWEL SIGN VOCALIC R */
778 0x1057, /* MYANMAR VOWEL SIGN VOCALIC RR */
779 0x1062, /* MYANMAR VOWEL SIGN SGAW KAREN EU */
780 0x1063, /* MYANMAR TONE MARK SGAW KAREN HATHI */
781 0x1064, /* MYANMAR TONE MARK SGAW KAREN KE PHO */
782 0x1067, /* MYANMAR VOWEL SIGN WESTERN PWO KAREN EU */
783 0x1068, /* MYANMAR VOWEL SIGN WESTERN PWO KAREN UE */
784 0x1069, /* MYANMAR SIGN WESTERN PWO KAREN TONE-1 */
785 0x106A, /* MYANMAR SIGN WESTERN PWO KAREN TONE-2 */
786 0x106B, /* MYANMAR SIGN WESTERN PWO KAREN TONE-3 */
787 0x106C, /* MYANMAR SIGN WESTERN PWO KAREN TONE-4 */
788 0x106D, /* MYANMAR SIGN WESTERN PWO KAREN TONE-5 */
789 0x1083, /* MYANMAR VOWEL SIGN SHAN AA */
790 0x1084, /* MYANMAR VOWEL SIGN SHAN E */
791 0x1087, /* MYANMAR SIGN SHAN TONE-2 */
792 0x1088, /* MYANMAR SIGN SHAN TONE-3 */
793 0x1089, /* MYANMAR SIGN SHAN TONE-5 */
794 0x108A, /* MYANMAR SIGN SHAN TONE-6 */
795 0x108B, /* MYANMAR SIGN SHAN COUNCIL TONE-2 */
796 0x108C, /* MYANMAR SIGN SHAN COUNCIL TONE-3 */
797 0x108F, /* MYANMAR SIGN RUMAI PALAUNG TONE-5 */
798 0x17B6, /* KHMER VOWEL SIGN AA */
799 0x17BE, /* KHMER VOWEL SIGN OE */
800 0x17BF, /* KHMER VOWEL SIGN YA */
801 0x17C0, /* KHMER VOWEL SIGN IE */
802 0x17C1, /* KHMER VOWEL SIGN E */
803 0x17C2, /* KHMER VOWEL SIGN AE */
804 0x17C3, /* KHMER VOWEL SIGN AI */
805 0x17C4, /* KHMER VOWEL SIGN OO */
806 0x17C5, /* KHMER VOWEL SIGN AU */
807 0x17C7, /* KHMER SIGN REAHMUK */
808 0x17C8, /* KHMER SIGN YUUKALEAPINTU */
809 0x1923, /* LIMBU VOWEL SIGN EE */
810 0x1924, /* LIMBU VOWEL SIGN AI */
811 0x1925, /* LIMBU VOWEL SIGN OO */
812 0x1926, /* LIMBU VOWEL SIGN AU */
813 0x1929, /* LIMBU SUBJOINED LETTER YA */
814 0x192A, /* LIMBU SUBJOINED LETTER RA */
815 0x192B, /* LIMBU SUBJOINED LETTER WA */
816 0x1930, /* LIMBU SMALL LETTER KA */
817 0x1931, /* LIMBU SMALL LETTER NGA */
818 0x1933, /* LIMBU SMALL LETTER TA */
819 0x1934, /* LIMBU SMALL LETTER NA */
820 0x1935, /* LIMBU SMALL LETTER PA */
821 0x1936, /* LIMBU SMALL LETTER MA */
822 0x1937, /* LIMBU SMALL LETTER RA */
823 0x1938, /* LIMBU SMALL LETTER LA */
824 0x19B0, /* NEW TAI LUE VOWEL SIGN VOWEL SHORTENER */
825 0x19B1, /* NEW TAI LUE VOWEL SIGN AA */
826 0x19B2, /* NEW TAI LUE VOWEL SIGN II */
827 0x19B3, /* NEW TAI LUE VOWEL SIGN U */
828 0x19B4, /* NEW TAI LUE VOWEL SIGN UU */
829 0x19B5, /* NEW TAI LUE VOWEL SIGN E */
830 0x19B6, /* NEW TAI LUE VOWEL SIGN AE */
831 0x19B7, /* NEW TAI LUE VOWEL SIGN O */
832 0x19B8, /* NEW TAI LUE VOWEL SIGN OA */
833 0x19B9, /* NEW TAI LUE VOWEL SIGN UE */
834 0x19BA, /* NEW TAI LUE VOWEL SIGN AY */
835 0x19BB, /* NEW TAI LUE VOWEL SIGN AAY */
836 0x19BC, /* NEW TAI LUE VOWEL SIGN UY */
837 0x19BD, /* NEW TAI LUE VOWEL SIGN OY */
838 0x19BE, /* NEW TAI LUE VOWEL SIGN OAY */
839 0x19BF, /* NEW TAI LUE VOWEL SIGN UEY */
840 0x19C0, /* NEW TAI LUE VOWEL SIGN IY */
841 0x19C8, /* NEW TAI LUE TONE MARK-1 */
842 0x19C9, /* NEW TAI LUE TONE MARK-2 */
843 0x1A19, /* BUGINESE VOWEL SIGN E */
844 0x1A1A, /* BUGINESE VOWEL SIGN O */
845 0x1A1B, /* BUGINESE VOWEL SIGN AE */
846 0x1B04, /* BALINESE SIGN BISAH */
847 0x1B35, /* BALINESE VOWEL SIGN TEDUNG */
848 0x1B3B, /* BALINESE VOWEL SIGN RA REPA TEDUNG */
849 0x1B3D, /* BALINESE VOWEL SIGN LA LENGA TEDUNG */
850 0x1B3E, /* BALINESE VOWEL SIGN TALING */
851 0x1B3F, /* BALINESE VOWEL SIGN TALING REPA */
852 0x1B40, /* BALINESE VOWEL SIGN TALING TEDUNG */
853 0x1B41, /* BALINESE VOWEL SIGN TALING REPA TEDUNG */
854 0x1B43, /* BALINESE VOWEL SIGN PEPET TEDUNG */
855 0x1B44, /* BALINESE ADEG ADEG */
856 0x1B82, /* SUNDANESE SIGN PANGWISAD */
857 0x1BA1, /* SUNDANESE CONSONANT SIGN PAMINGKAL */
858 0x1BA6, /* SUNDANESE VOWEL SIGN PANAELAENG */
859 0x1BA7, /* SUNDANESE VOWEL SIGN PANOLONG */
860 0x1BAA, /* SUNDANESE SIGN PAMAAEH */
861 0x1C24, /* LEPCHA SUBJOINED LETTER YA */
862 0x1C25, /* LEPCHA SUBJOINED LETTER RA */
863 0x1C26, /* LEPCHA VOWEL SIGN AA */
864 0x1C27, /* LEPCHA VOWEL SIGN I */
865 0x1C28, /* LEPCHA VOWEL SIGN O */
866 0x1C29, /* LEPCHA VOWEL SIGN OO */
867 0x1C2A, /* LEPCHA VOWEL SIGN U */
868 0x1C2B, /* LEPCHA VOWEL SIGN UU */
869 0x1C34, /* LEPCHA CONSONANT SIGN NYIN-DO */
870 0x1C35, /* LEPCHA CONSONANT SIGN KANG */
871 0xA823, /* SYLOTI NAGRI VOWEL SIGN A */
872 0xA824, /* SYLOTI NAGRI VOWEL SIGN I */
873 0xA827, /* SYLOTI NAGRI VOWEL SIGN OO */
874 0xA880, /* SAURASHTRA SIGN ANUSVARA */
875 0xA881, /* SAURASHTRA SIGN VISARGA */
876 0xA8B4, /* SAURASHTRA CONSONANT SIGN HAARU */
877 0xA8B5, /* SAURASHTRA VOWEL SIGN AA */
878 0xA8B6, /* SAURASHTRA VOWEL SIGN I */
879 0xA8B7, /* SAURASHTRA VOWEL SIGN II */
880 0xA8B8, /* SAURASHTRA VOWEL SIGN U */
881 0xA8B9, /* SAURASHTRA VOWEL SIGN UU */
882 0xA8BA, /* SAURASHTRA VOWEL SIGN VOCALIC R */
883 0xA8BB, /* SAURASHTRA VOWEL SIGN VOCALIC RR */
884 0xA8BC, /* SAURASHTRA VOWEL SIGN VOCALIC L */
885 0xA8BD, /* SAURASHTRA VOWEL SIGN VOCALIC LL */
886 0xA8BE, /* SAURASHTRA VOWEL SIGN E */
887 0xA8BF, /* SAURASHTRA VOWEL SIGN EE */
888 0xA8C0, /* SAURASHTRA VOWEL SIGN AI */
889 0xA8C1, /* SAURASHTRA VOWEL SIGN O */
890 0xA8C2, /* SAURASHTRA VOWEL SIGN OO */
891 0xA8C3, /* SAURASHTRA VOWEL SIGN AU */
892 0xA952, /* REJANG CONSONANT SIGN H */
893 0xA953, /* REJANG VIRAMA */
894 0xAA2F, /* CHAM VOWEL SIGN O */
895 0xAA30, /* CHAM VOWEL SIGN AI */
896 0xAA33, /* CHAM CONSONANT SIGN YA */
897 0xAA34, /* CHAM CONSONANT SIGN RA */
898 0xAA4D /* CHAM CONSONANT SIGN FINAL H */
899 };
900 const pg_wchar *StopLow = strange_letter,
901 *StopHigh = strange_letter + lengthof(strange_letter),
902 *StopMiddle;
903 pg_wchar c;
904
905 c = *(prs->pgwstr + prs->state->poschar);
906
907 while (StopLow < StopHigh)
908 {
909 StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
910 if (*StopMiddle == c)
911 return 1;
912 else if (*StopMiddle < c)
913 StopLow = StopMiddle + 1;
914 else
915 StopHigh = StopMiddle;
916 }
917 }
918
919 return 0;
920}
921
922/*
923 * Table of state/action of parser
924 */
925
926static const TParserStateActionItem actionTPS_Base[] = {
927 {p_isEOF, 0, A_NEXT, TPS_Null, 0, NULL},
928 {p_iseqC, '<', A_PUSH, TPS_InTagFirst, 0, NULL},
929 {p_isignore, 0, A_NEXT, TPS_InSpace, 0, NULL},
930 {p_isasclet, 0, A_NEXT, TPS_InAsciiWord, 0, NULL},
931 {p_isalpha, 0, A_NEXT, TPS_InWord, 0, NULL},
932 {p_isdigit, 0, A_NEXT, TPS_InUnsignedInt, 0, NULL},
933 {p_iseqC, '-', A_PUSH, TPS_InSignedIntFirst, 0, NULL},
934 {p_iseqC, '+', A_PUSH, TPS_InSignedIntFirst, 0, NULL},
935 {p_iseqC, '&', A_PUSH, TPS_InXMLEntityFirst, 0, NULL},
936 {p_iseqC, '~', A_PUSH, TPS_InFileTwiddle, 0, NULL},
937 {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL},
938 {p_iseqC, '.', A_PUSH, TPS_InPathFirstFirst, 0, NULL},
939 {NULL, 0, A_NEXT, TPS_InSpace, 0, NULL}
940};
941
942
943static const TParserStateActionItem actionTPS_InNumWord[] = {
944 {p_isEOF, 0, A_BINGO, TPS_Base, NUMWORD, NULL},
945 {p_isalnum, 0, A_NEXT, TPS_InNumWord, 0, NULL},
946 {p_isspecial, 0, A_NEXT, TPS_InNumWord, 0, NULL},
947 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
948 {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL},
949 {p_iseqC, '.', A_PUSH, TPS_InFileNext, 0, NULL},
950 {p_iseqC, '-', A_PUSH, TPS_InHyphenNumWordFirst, 0, NULL},
951 {NULL, 0, A_BINGO, TPS_Base, NUMWORD, NULL}
952};
953
954static const TParserStateActionItem actionTPS_InAsciiWord[] = {
955 {p_isEOF, 0, A_BINGO, TPS_Base, ASCIIWORD, NULL},
956 {p_isasclet, 0, A_NEXT, TPS_Null, 0, NULL},
957 {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL},
958 {p_iseqC, '.', A_PUSH, TPS_InFileNext, 0, NULL},
959 {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL},
960 {p_iseqC, '-', A_PUSH, TPS_InHyphenAsciiWordFirst, 0, NULL},
961 {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL},
962 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
963 {p_iseqC, ':', A_PUSH, TPS_InProtocolFirst, 0, NULL},
964 {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL},
965 {p_isdigit, 0, A_PUSH, TPS_InHost, 0, NULL},
966 {p_isdigit, 0, A_NEXT, TPS_InNumWord, 0, NULL},
967 {p_isalpha, 0, A_NEXT, TPS_InWord, 0, NULL},
968 {p_isspecial, 0, A_NEXT, TPS_InWord, 0, NULL},
969 {NULL, 0, A_BINGO, TPS_Base, ASCIIWORD, NULL}
970};
971
972static const TParserStateActionItem actionTPS_InWord[] = {
973 {p_isEOF, 0, A_BINGO, TPS_Base, WORD_T, NULL},
974 {p_isalpha, 0, A_NEXT, TPS_Null, 0, NULL},
975 {p_isspecial, 0, A_NEXT, TPS_Null, 0, NULL},
976 {p_isdigit, 0, A_NEXT, TPS_InNumWord, 0, NULL},
977 {p_iseqC, '-', A_PUSH, TPS_InHyphenWordFirst, 0, NULL},
978 {NULL, 0, A_BINGO, TPS_Base, WORD_T, NULL}
979};
980
981static const TParserStateActionItem actionTPS_InUnsignedInt[] = {
982 {p_isEOF, 0, A_BINGO, TPS_Base, UNSIGNEDINT, NULL},
983 {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL},
984 {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL},
985 {p_iseqC, '.', A_PUSH, TPS_InUDecimalFirst, 0, NULL},
986 {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL},
987 {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL},
988 {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL},
989 {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL},
990 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
991 {p_isasclet, 0, A_PUSH, TPS_InHost, 0, NULL},
992 {p_isalpha, 0, A_NEXT, TPS_InNumWord, 0, NULL},
993 {p_isspecial, 0, A_NEXT, TPS_InNumWord, 0, NULL},
994 {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL},
995 {NULL, 0, A_BINGO, TPS_Base, UNSIGNEDINT, NULL}
996};
997
998static const TParserStateActionItem actionTPS_InSignedIntFirst[] = {
999 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1000 {p_isdigit, 0, A_NEXT | A_CLEAR, TPS_InSignedInt, 0, NULL},
1001 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1002};
1003
1004static const TParserStateActionItem actionTPS_InSignedInt[] = {
1005 {p_isEOF, 0, A_BINGO, TPS_Base, SIGNEDINT, NULL},
1006 {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL},
1007 {p_iseqC, '.', A_PUSH, TPS_InDecimalFirst, 0, NULL},
1008 {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1009 {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1010 {NULL, 0, A_BINGO, TPS_Base, SIGNEDINT, NULL}
1011};
1012
1013static const TParserStateActionItem actionTPS_InSpace[] = {
1014 {p_isEOF, 0, A_BINGO, TPS_Base, SPACE, NULL},
1015 {p_iseqC, '<', A_BINGO, TPS_Base, SPACE, NULL},
1016 {p_isignore, 0, A_NEXT, TPS_Null, 0, NULL},
1017 {p_iseqC, '-', A_BINGO, TPS_Base, SPACE, NULL},
1018 {p_iseqC, '+', A_BINGO, TPS_Base, SPACE, NULL},
1019 {p_iseqC, '&', A_BINGO, TPS_Base, SPACE, NULL},
1020 {p_iseqC, '/', A_BINGO, TPS_Base, SPACE, NULL},
1021 {p_isnotalnum, 0, A_NEXT, TPS_InSpace, 0, NULL},
1022 {NULL, 0, A_BINGO, TPS_Base, SPACE, NULL}
1023};
1024
1025static const TParserStateActionItem actionTPS_InUDecimalFirst[] = {
1026 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1027 {p_isdigit, 0, A_CLEAR, TPS_InUDecimal, 0, NULL},
1028 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1029};
1030
1031static const TParserStateActionItem actionTPS_InUDecimal[] = {
1032 {p_isEOF, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL},
1033 {p_isdigit, 0, A_NEXT, TPS_InUDecimal, 0, NULL},
1034 {p_iseqC, '.', A_PUSH, TPS_InVersionFirst, 0, NULL},
1035 {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1036 {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1037 {NULL, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL}
1038};
1039
1040static const TParserStateActionItem actionTPS_InDecimalFirst[] = {
1041 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1042 {p_isdigit, 0, A_CLEAR, TPS_InDecimal, 0, NULL},
1043 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1044};
1045
1046static const TParserStateActionItem actionTPS_InDecimal[] = {
1047 {p_isEOF, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL},
1048 {p_isdigit, 0, A_NEXT, TPS_InDecimal, 0, NULL},
1049 {p_iseqC, '.', A_PUSH, TPS_InVerVersion, 0, NULL},
1050 {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1051 {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1052 {NULL, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL}
1053};
1054
1055static const TParserStateActionItem actionTPS_InVerVersion[] = {
1056 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1057 {p_isdigit, 0, A_RERUN, TPS_InSVerVersion, 0, SpecialVerVersion},
1058 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1059};
1060
1061static const TParserStateActionItem actionTPS_InSVerVersion[] = {
1062 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1063 {p_isdigit, 0, A_BINGO | A_CLRALL, TPS_InUnsignedInt, SPACE, NULL},
1064 {NULL, 0, A_NEXT, TPS_Null, 0, NULL}
1065};
1066
1067
1068static const TParserStateActionItem actionTPS_InVersionFirst[] = {
1069 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1070 {p_isdigit, 0, A_CLEAR, TPS_InVersion, 0, NULL},
1071 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1072};
1073
1074static const TParserStateActionItem actionTPS_InVersion[] = {
1075 {p_isEOF, 0, A_BINGO, TPS_Base, VERSIONNUMBER, NULL},
1076 {p_isdigit, 0, A_NEXT, TPS_InVersion, 0, NULL},
1077 {p_iseqC, '.', A_PUSH, TPS_InVersionFirst, 0, NULL},
1078 {NULL, 0, A_BINGO, TPS_Base, VERSIONNUMBER, NULL}
1079};
1080
1081static const TParserStateActionItem actionTPS_InMantissaFirst[] = {
1082 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1083 {p_isdigit, 0, A_CLEAR, TPS_InMantissa, 0, NULL},
1084 {p_iseqC, '+', A_NEXT, TPS_InMantissaSign, 0, NULL},
1085 {p_iseqC, '-', A_NEXT, TPS_InMantissaSign, 0, NULL},
1086 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1087};
1088
1089static const TParserStateActionItem actionTPS_InMantissaSign[] = {
1090 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1091 {p_isdigit, 0, A_CLEAR, TPS_InMantissa, 0, NULL},
1092 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1093};
1094
1095static const TParserStateActionItem actionTPS_InMantissa[] = {
1096 {p_isEOF, 0, A_BINGO, TPS_Base, SCIENTIFIC, NULL},
1097 {p_isdigit, 0, A_NEXT, TPS_InMantissa, 0, NULL},
1098 {NULL, 0, A_BINGO, TPS_Base, SCIENTIFIC, NULL}
1099};
1100
1101static const TParserStateActionItem actionTPS_InXMLEntityFirst[] = {
1102 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1103 {p_iseqC, '#', A_NEXT, TPS_InXMLEntityNumFirst, 0, NULL},
1104 {p_isasclet, 0, A_NEXT, TPS_InXMLEntity, 0, NULL},
1105 {p_iseqC, ':', A_NEXT, TPS_InXMLEntity, 0, NULL},
1106 {p_iseqC, '_', A_NEXT, TPS_InXMLEntity, 0, NULL},
1107 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1108};
1109
1110static const TParserStateActionItem actionTPS_InXMLEntity[] = {
1111 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1112 {p_isalnum, 0, A_NEXT, TPS_InXMLEntity, 0, NULL},
1113 {p_iseqC, ':', A_NEXT, TPS_InXMLEntity, 0, NULL},
1114 {p_iseqC, '_', A_NEXT, TPS_InXMLEntity, 0, NULL},
1115 {p_iseqC, '.', A_NEXT, TPS_InXMLEntity, 0, NULL},
1116 {p_iseqC, '-', A_NEXT, TPS_InXMLEntity, 0, NULL},
1117 {p_iseqC, ';', A_NEXT, TPS_InXMLEntityEnd, 0, NULL},
1118 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1119};
1120
1121static const TParserStateActionItem actionTPS_InXMLEntityNumFirst[] = {
1122 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1123 {p_iseqC, 'x', A_NEXT, TPS_InXMLEntityHexNumFirst, 0, NULL},
1124 {p_iseqC, 'X', A_NEXT, TPS_InXMLEntityHexNumFirst, 0, NULL},
1125 {p_isdigit, 0, A_NEXT, TPS_InXMLEntityNum, 0, NULL},
1126 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1127};
1128
1129static const TParserStateActionItem actionTPS_InXMLEntityHexNumFirst[] = {
1130 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1131 {p_isxdigit, 0, A_NEXT, TPS_InXMLEntityHexNum, 0, NULL},
1132 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1133};
1134
1135static const TParserStateActionItem actionTPS_InXMLEntityNum[] = {
1136 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1137 {p_isdigit, 0, A_NEXT, TPS_InXMLEntityNum, 0, NULL},
1138 {p_iseqC, ';', A_NEXT, TPS_InXMLEntityEnd, 0, NULL},
1139 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1140};
1141
1142static const TParserStateActionItem actionTPS_InXMLEntityHexNum[] = {
1143 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1144 {p_isxdigit, 0, A_NEXT, TPS_InXMLEntityHexNum, 0, NULL},
1145 {p_iseqC, ';', A_NEXT, TPS_InXMLEntityEnd, 0, NULL},
1146 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1147};
1148
1149static const TParserStateActionItem actionTPS_InXMLEntityEnd[] = {
1150 {NULL, 0, A_BINGO | A_CLEAR, TPS_Base, XMLENTITY, NULL}
1151};
1152
1153static const TParserStateActionItem actionTPS_InTagFirst[] = {
1154 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1155 {p_iseqC, '/', A_PUSH, TPS_InTagCloseFirst, 0, NULL},
1156 {p_iseqC, '!', A_PUSH, TPS_InCommentFirst, 0, NULL},
1157 {p_iseqC, '?', A_PUSH, TPS_InXMLBegin, 0, NULL},
1158 {p_isasclet, 0, A_PUSH, TPS_InTagName, 0, NULL},
1159 {p_iseqC, ':', A_PUSH, TPS_InTagName, 0, NULL},
1160 {p_iseqC, '_', A_PUSH, TPS_InTagName, 0, NULL},
1161 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1162};
1163
1164static const TParserStateActionItem actionTPS_InXMLBegin[] = {
1165 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1166 /* <?xml ... */
1167 /* XXX do we wants states for the m and l ? Right now this accepts <?xZ */
1168 {p_iseqC, 'x', A_NEXT, TPS_InTag, 0, NULL},
1169 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1170};
1171
1172static const TParserStateActionItem actionTPS_InTagCloseFirst[] = {
1173 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1174 {p_isasclet, 0, A_NEXT, TPS_InTagName, 0, NULL},
1175 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1176};
1177
1178static const TParserStateActionItem actionTPS_InTagName[] = {
1179 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1180 /* <br/> case */
1181 {p_iseqC, '/', A_NEXT, TPS_InTagBeginEnd, 0, NULL},
1182 {p_iseqC, '>', A_NEXT, TPS_InTagEnd, 0, SpecialTags},
1183 {p_isspace, 0, A_NEXT, TPS_InTag, 0, SpecialTags},
1184 {p_isalnum, 0, A_NEXT, TPS_Null, 0, NULL},
1185 {p_iseqC, ':', A_NEXT, TPS_Null, 0, NULL},
1186 {p_iseqC, '_', A_NEXT, TPS_Null, 0, NULL},
1187 {p_iseqC, '.', A_NEXT, TPS_Null, 0, NULL},
1188 {p_iseqC, '-', A_NEXT, TPS_Null, 0, NULL},
1189 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1190};
1191
1192static const TParserStateActionItem actionTPS_InTagBeginEnd[] = {
1193 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1194 {p_iseqC, '>', A_NEXT, TPS_InTagEnd, 0, NULL},
1195 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1196};
1197
1198static const TParserStateActionItem actionTPS_InTag[] = {
1199 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1200 {p_iseqC, '>', A_NEXT, TPS_InTagEnd, 0, SpecialTags},
1201 {p_iseqC, '\'', A_NEXT, TPS_InTagEscapeK, 0, NULL},
1202 {p_iseqC, '"', A_NEXT, TPS_InTagEscapeKK, 0, NULL},
1203 {p_isasclet, 0, A_NEXT, TPS_Null, 0, NULL},
1204 {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL},
1205 {p_iseqC, '=', A_NEXT, TPS_Null, 0, NULL},
1206 {p_iseqC, '-', A_NEXT, TPS_Null, 0, NULL},
1207 {p_iseqC, '_', A_NEXT, TPS_Null, 0, NULL},
1208 {p_iseqC, '#', A_NEXT, TPS_Null, 0, NULL},
1209 {p_iseqC, '/', A_NEXT, TPS_Null, 0, NULL},
1210 {p_iseqC, ':', A_NEXT, TPS_Null, 0, NULL},
1211 {p_iseqC, '.', A_NEXT, TPS_Null, 0, NULL},
1212 {p_iseqC, '&', A_NEXT, TPS_Null, 0, NULL},
1213 {p_iseqC, '?', A_NEXT, TPS_Null, 0, NULL},
1214 {p_iseqC, '%', A_NEXT, TPS_Null, 0, NULL},
1215 {p_iseqC, '~', A_NEXT, TPS_Null, 0, NULL},
1216 {p_isspace, 0, A_NEXT, TPS_Null, 0, SpecialTags},
1217 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1218};
1219
1220static const TParserStateActionItem actionTPS_InTagEscapeK[] = {
1221 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1222 {p_iseqC, '\\', A_PUSH, TPS_InTagBackSleshed, 0, NULL},
1223 {p_iseqC, '\'', A_NEXT, TPS_InTag, 0, NULL},
1224 {NULL, 0, A_NEXT, TPS_InTagEscapeK, 0, NULL}
1225};
1226
1227static const TParserStateActionItem actionTPS_InTagEscapeKK[] = {
1228 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1229 {p_iseqC, '\\', A_PUSH, TPS_InTagBackSleshed, 0, NULL},
1230 {p_iseqC, '"', A_NEXT, TPS_InTag, 0, NULL},
1231 {NULL, 0, A_NEXT, TPS_InTagEscapeKK, 0, NULL}
1232};
1233
1234static const TParserStateActionItem actionTPS_InTagBackSleshed[] = {
1235 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1236 {NULL, 0, A_MERGE, TPS_Null, 0, NULL}
1237};
1238
1239static const TParserStateActionItem actionTPS_InTagEnd[] = {
1240 {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, TAG_T, NULL}
1241};
1242
1243static const TParserStateActionItem actionTPS_InCommentFirst[] = {
1244 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1245 {p_iseqC, '-', A_NEXT, TPS_InCommentLast, 0, NULL},
1246 /* <!DOCTYPE ...> */
1247 {p_iseqC, 'D', A_NEXT, TPS_InTag, 0, NULL},
1248 {p_iseqC, 'd', A_NEXT, TPS_InTag, 0, NULL},
1249 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1250};
1251
1252static const TParserStateActionItem actionTPS_InCommentLast[] = {
1253 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1254 {p_iseqC, '-', A_NEXT, TPS_InComment, 0, NULL},
1255 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1256};
1257
1258static const TParserStateActionItem actionTPS_InComment[] = {
1259 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1260 {p_iseqC, '-', A_NEXT, TPS_InCloseCommentFirst, 0, NULL},
1261 {NULL, 0, A_NEXT, TPS_Null, 0, NULL}
1262};
1263
1264static const TParserStateActionItem actionTPS_InCloseCommentFirst[] = {
1265 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1266 {p_iseqC, '-', A_NEXT, TPS_InCloseCommentLast, 0, NULL},
1267 {NULL, 0, A_NEXT, TPS_InComment, 0, NULL}
1268};
1269
1270static const TParserStateActionItem actionTPS_InCloseCommentLast[] = {
1271 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1272 {p_iseqC, '-', A_NEXT, TPS_Null, 0, NULL},
1273 {p_iseqC, '>', A_NEXT, TPS_InCommentEnd, 0, NULL},
1274 {NULL, 0, A_NEXT, TPS_InComment, 0, NULL}
1275};
1276
1277static const TParserStateActionItem actionTPS_InCommentEnd[] = {
1278 {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, TAG_T, NULL}
1279};
1280
1281static const TParserStateActionItem actionTPS_InHostFirstDomain[] = {
1282 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1283 {p_isasclet, 0, A_NEXT, TPS_InHostDomainSecond, 0, NULL},
1284 {p_isdigit, 0, A_NEXT, TPS_InHost, 0, NULL},
1285 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1286};
1287
1288static const TParserStateActionItem actionTPS_InHostDomainSecond[] = {
1289 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1290 {p_isasclet, 0, A_NEXT, TPS_InHostDomain, 0, NULL},
1291 {p_isdigit, 0, A_PUSH, TPS_InHost, 0, NULL},
1292 {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1293 {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1294 {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL},
1295 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
1296 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1297};
1298
1299static const TParserStateActionItem actionTPS_InHostDomain[] = {
1300 {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL},
1301 {p_isasclet, 0, A_NEXT, TPS_InHostDomain, 0, NULL},
1302 {p_isdigit, 0, A_PUSH, TPS_InHost, 0, NULL},
1303 {p_iseqC, ':', A_PUSH, TPS_InPortFirst, 0, NULL},
1304 {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1305 {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1306 {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL},
1307 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
1308 {p_isdigit, 0, A_POP, TPS_Null, 0, NULL},
1309 {p_isstophost, 0, A_BINGO | A_CLRALL, TPS_InURLPathStart, HOST, NULL},
1310 {p_iseqC, '/', A_PUSH, TPS_InFURL, 0, NULL},
1311 {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL}
1312};
1313
1314static const TParserStateActionItem actionTPS_InPortFirst[] = {
1315 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1316 {p_isdigit, 0, A_NEXT, TPS_InPort, 0, NULL},
1317 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1318};
1319
1320static const TParserStateActionItem actionTPS_InPort[] = {
1321 {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL},
1322 {p_isdigit, 0, A_NEXT, TPS_InPort, 0, NULL},
1323 {p_isstophost, 0, A_BINGO | A_CLRALL, TPS_InURLPathStart, HOST, NULL},
1324 {p_iseqC, '/', A_PUSH, TPS_InFURL, 0, NULL},
1325 {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL}
1326};
1327
1328static const TParserStateActionItem actionTPS_InHostFirstAN[] = {
1329 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1330 {p_isdigit, 0, A_NEXT, TPS_InHost, 0, NULL},
1331 {p_isasclet, 0, A_NEXT, TPS_InHost, 0, NULL},
1332 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1333};
1334
1335static const TParserStateActionItem actionTPS_InHost[] = {
1336 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1337 {p_isdigit, 0, A_NEXT, TPS_InHost, 0, NULL},
1338 {p_isasclet, 0, A_NEXT, TPS_InHost, 0, NULL},
1339 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
1340 {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL},
1341 {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1342 {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1343 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1344};
1345
1346static const TParserStateActionItem actionTPS_InEmail[] = {
1347 {p_isstophost, 0, A_POP, TPS_Null, 0, NULL},
1348 {p_ishost, 0, A_BINGO | A_CLRALL, TPS_Base, EMAIL, NULL},
1349 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1350};
1351
1352static const TParserStateActionItem actionTPS_InFileFirst[] = {
1353 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1354 {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL},
1355 {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL},
1356 {p_iseqC, '.', A_NEXT, TPS_InPathFirst, 0, NULL},
1357 {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL},
1358 {p_iseqC, '~', A_PUSH, TPS_InFileTwiddle, 0, NULL},
1359 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1360};
1361
1362static const TParserStateActionItem actionTPS_InFileTwiddle[] = {
1363 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1364 {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL},
1365 {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL},
1366 {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL},
1367 {p_iseqC, '/', A_NEXT, TPS_InFileFirst, 0, NULL},
1368 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1369};
1370
1371static const TParserStateActionItem actionTPS_InPathFirst[] = {
1372 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1373 {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL},
1374 {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL},
1375 {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL},
1376 {p_iseqC, '.', A_NEXT, TPS_InPathSecond, 0, NULL},
1377 {p_iseqC, '/', A_NEXT, TPS_InFileFirst, 0, NULL},
1378 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1379};
1380
1381static const TParserStateActionItem actionTPS_InPathFirstFirst[] = {
1382 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1383 {p_iseqC, '.', A_NEXT, TPS_InPathSecond, 0, NULL},
1384 {p_iseqC, '/', A_NEXT, TPS_InFileFirst, 0, NULL},
1385 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1386};
1387
1388static const TParserStateActionItem actionTPS_InPathSecond[] = {
1389 {p_isEOF, 0, A_BINGO | A_CLEAR, TPS_Base, FILEPATH, NULL},
1390 {p_iseqC, '/', A_NEXT | A_PUSH, TPS_InFileFirst, 0, NULL},
1391 {p_iseqC, '/', A_BINGO | A_CLEAR, TPS_Base, FILEPATH, NULL},
1392 {p_isspace, 0, A_BINGO | A_CLEAR, TPS_Base, FILEPATH, NULL},
1393 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1394};
1395
1396static const TParserStateActionItem actionTPS_InFile[] = {
1397 {p_isEOF, 0, A_BINGO, TPS_Base, FILEPATH, NULL},
1398 {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL},
1399 {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL},
1400 {p_iseqC, '.', A_PUSH, TPS_InFileNext, 0, NULL},
1401 {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL},
1402 {p_iseqC, '-', A_NEXT, TPS_InFile, 0, NULL},
1403 {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL},
1404 {NULL, 0, A_BINGO, TPS_Base, FILEPATH, NULL}
1405};
1406
1407static const TParserStateActionItem actionTPS_InFileNext[] = {
1408 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1409 {p_isasclet, 0, A_CLEAR, TPS_InFile, 0, NULL},
1410 {p_isdigit, 0, A_CLEAR, TPS_InFile, 0, NULL},
1411 {p_iseqC, '_', A_CLEAR, TPS_InFile, 0, NULL},
1412 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1413};
1414
1415static const TParserStateActionItem actionTPS_InURLPathFirst[] = {
1416 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1417 {p_isurlchar, 0, A_NEXT, TPS_InURLPath, 0, NULL},
1418 {NULL, 0, A_POP, TPS_Null, 0, NULL},
1419};
1420
1421static const TParserStateActionItem actionTPS_InURLPathStart[] = {
1422 {NULL, 0, A_NEXT, TPS_InURLPath, 0, NULL}
1423};
1424
1425static const TParserStateActionItem actionTPS_InURLPath[] = {
1426 {p_isEOF, 0, A_BINGO, TPS_Base, URLPATH, NULL},
1427 {p_isurlchar, 0, A_NEXT, TPS_InURLPath, 0, NULL},
1428 {NULL, 0, A_BINGO, TPS_Base, URLPATH, NULL}
1429};
1430
1431static const TParserStateActionItem actionTPS_InFURL[] = {
1432 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1433 {p_isURLPath, 0, A_BINGO | A_CLRALL, TPS_Base, URL_T, SpecialFURL},
1434 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1435};
1436
1437static const TParserStateActionItem actionTPS_InProtocolFirst[] = {
1438 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1439 {p_iseqC, '/', A_NEXT, TPS_InProtocolSecond, 0, NULL},
1440 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1441};
1442
1443static const TParserStateActionItem actionTPS_InProtocolSecond[] = {
1444 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1445 {p_iseqC, '/', A_NEXT, TPS_InProtocolEnd, 0, NULL},
1446 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1447};
1448
1449static const TParserStateActionItem actionTPS_InProtocolEnd[] = {
1450 {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, PROTOCOL, NULL}
1451};
1452
1453static const TParserStateActionItem actionTPS_InHyphenAsciiWordFirst[] = {
1454 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1455 {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWord, 0, NULL},
1456 {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1457 {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL},
1458 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1459};
1460
1461static const TParserStateActionItem actionTPS_InHyphenAsciiWord[] = {
1462 {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, ASCIIHWORD, SpecialHyphen},
1463 {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWord, 0, NULL},
1464 {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1465 {p_isspecial, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1466 {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1467 {p_iseqC, '-', A_PUSH, TPS_InHyphenAsciiWordFirst, 0, NULL},
1468 {NULL, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, ASCIIHWORD, SpecialHyphen}
1469};
1470
1471static const TParserStateActionItem actionTPS_InHyphenWordFirst[] = {
1472 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1473 {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1474 {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL},
1475 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1476};
1477
1478static const TParserStateActionItem actionTPS_InHyphenWord[] = {
1479 {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, HWORD, SpecialHyphen},
1480 {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1481 {p_isspecial, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1482 {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1483 {p_iseqC, '-', A_PUSH, TPS_InHyphenWordFirst, 0, NULL},
1484 {NULL, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, HWORD, SpecialHyphen}
1485};
1486
1487static const TParserStateActionItem actionTPS_InHyphenNumWordFirst[] = {
1488 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1489 {p_isalpha, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1490 {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL},
1491 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1492};
1493
1494static const TParserStateActionItem actionTPS_InHyphenNumWord[] = {
1495 {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, NUMHWORD, SpecialHyphen},
1496 {p_isalnum, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1497 {p_isspecial, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1498 {p_iseqC, '-', A_PUSH, TPS_InHyphenNumWordFirst, 0, NULL},
1499 {NULL, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, NUMHWORD, SpecialHyphen}
1500};
1501
1502static const TParserStateActionItem actionTPS_InHyphenDigitLookahead[] = {
1503 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1504 {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL},
1505 {p_isalpha, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1506 {p_isspecial, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1507 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1508};
1509
1510static const TParserStateActionItem actionTPS_InParseHyphen[] = {
1511 {p_isEOF, 0, A_RERUN, TPS_Base, 0, NULL},
1512 {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWordPart, 0, NULL},
1513 {p_isalpha, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL},
1514 {p_isdigit, 0, A_PUSH, TPS_InHyphenUnsignedInt, 0, NULL},
1515 {p_iseqC, '-', A_PUSH, TPS_InParseHyphenHyphen, 0, NULL},
1516 {NULL, 0, A_RERUN, TPS_Base, 0, NULL}
1517};
1518
1519static const TParserStateActionItem actionTPS_InParseHyphenHyphen[] = {
1520 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1521 {p_isalnum, 0, A_BINGO | A_CLEAR, TPS_InParseHyphen, SPACE, NULL},
1522 {p_isspecial, 0, A_BINGO | A_CLEAR, TPS_InParseHyphen, SPACE, NULL},
1523 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1524};
1525
1526static const TParserStateActionItem actionTPS_InHyphenWordPart[] = {
1527 {p_isEOF, 0, A_BINGO, TPS_Base, PARTHWORD, NULL},
1528 {p_isalpha, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL},
1529 {p_isspecial, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL},
1530 {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL},
1531 {NULL, 0, A_BINGO, TPS_InParseHyphen, PARTHWORD, NULL}
1532};
1533
1534static const TParserStateActionItem actionTPS_InHyphenAsciiWordPart[] = {
1535 {p_isEOF, 0, A_BINGO, TPS_Base, ASCIIPARTHWORD, NULL},
1536 {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWordPart, 0, NULL},
1537 {p_isalpha, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL},
1538 {p_isspecial, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL},
1539 {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL},
1540 {NULL, 0, A_BINGO, TPS_InParseHyphen, ASCIIPARTHWORD, NULL}
1541};
1542
1543static const TParserStateActionItem actionTPS_InHyphenNumWordPart[] = {
1544 {p_isEOF, 0, A_BINGO, TPS_Base, NUMPARTHWORD, NULL},
1545 {p_isalnum, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL},
1546 {p_isspecial, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL},
1547 {NULL, 0, A_BINGO, TPS_InParseHyphen, NUMPARTHWORD, NULL}
1548};
1549
1550static const TParserStateActionItem actionTPS_InHyphenUnsignedInt[] = {
1551 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1552 {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL},
1553 {p_isalpha, 0, A_CLEAR, TPS_InHyphenNumWordPart, 0, NULL},
1554 {p_isspecial, 0, A_CLEAR, TPS_InHyphenNumWordPart, 0, NULL},
1555 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1556};
1557
1558
1559/*
1560 * main table of per-state parser actions
1561 */
1562typedef struct
1563{
1564 const TParserStateActionItem *action; /* the actual state info */
1565 TParserState state; /* only for Assert crosscheck */
1566#ifdef WPARSER_TRACE
1567 const char *state_name; /* only for debug printout */
1568#endif
1569} TParserStateAction;
1570
1571#ifdef WPARSER_TRACE
1572#define TPARSERSTATEACTION(state) \
1573 { CppConcat(action,state), state, CppAsString(state) }
1574#else
1575#define TPARSERSTATEACTION(state) \
1576 { CppConcat(action,state), state }
1577#endif
1578
1579/*
1580 * order must be the same as in typedef enum {} TParserState!!
1581 */
1582
1583static const TParserStateAction Actions[] = {
1584 TPARSERSTATEACTION(TPS_Base),
1585 TPARSERSTATEACTION(TPS_InNumWord),
1586 TPARSERSTATEACTION(TPS_InAsciiWord),
1587 TPARSERSTATEACTION(TPS_InWord),
1588 TPARSERSTATEACTION(TPS_InUnsignedInt),
1589 TPARSERSTATEACTION(TPS_InSignedIntFirst),
1590 TPARSERSTATEACTION(TPS_InSignedInt),
1591 TPARSERSTATEACTION(TPS_InSpace),
1592 TPARSERSTATEACTION(TPS_InUDecimalFirst),
1593 TPARSERSTATEACTION(TPS_InUDecimal),
1594 TPARSERSTATEACTION(TPS_InDecimalFirst),
1595 TPARSERSTATEACTION(TPS_InDecimal),
1596 TPARSERSTATEACTION(TPS_InVerVersion),
1597 TPARSERSTATEACTION(TPS_InSVerVersion),
1598 TPARSERSTATEACTION(TPS_InVersionFirst),
1599 TPARSERSTATEACTION(TPS_InVersion),
1600 TPARSERSTATEACTION(TPS_InMantissaFirst),
1601 TPARSERSTATEACTION(TPS_InMantissaSign),
1602 TPARSERSTATEACTION(TPS_InMantissa),
1603 TPARSERSTATEACTION(TPS_InXMLEntityFirst),
1604 TPARSERSTATEACTION(TPS_InXMLEntity),
1605 TPARSERSTATEACTION(TPS_InXMLEntityNumFirst),
1606 TPARSERSTATEACTION(TPS_InXMLEntityNum),
1607 TPARSERSTATEACTION(TPS_InXMLEntityHexNumFirst),
1608 TPARSERSTATEACTION(TPS_InXMLEntityHexNum),
1609 TPARSERSTATEACTION(TPS_InXMLEntityEnd),
1610 TPARSERSTATEACTION(TPS_InTagFirst),
1611 TPARSERSTATEACTION(TPS_InXMLBegin),
1612 TPARSERSTATEACTION(TPS_InTagCloseFirst),
1613 TPARSERSTATEACTION(TPS_InTagName),
1614 TPARSERSTATEACTION(TPS_InTagBeginEnd),
1615 TPARSERSTATEACTION(TPS_InTag),
1616 TPARSERSTATEACTION(TPS_InTagEscapeK),
1617 TPARSERSTATEACTION(TPS_InTagEscapeKK),
1618 TPARSERSTATEACTION(TPS_InTagBackSleshed),
1619 TPARSERSTATEACTION(TPS_InTagEnd),
1620 TPARSERSTATEACTION(TPS_InCommentFirst),
1621 TPARSERSTATEACTION(TPS_InCommentLast),
1622 TPARSERSTATEACTION(TPS_InComment),
1623 TPARSERSTATEACTION(TPS_InCloseCommentFirst),
1624 TPARSERSTATEACTION(TPS_InCloseCommentLast),
1625 TPARSERSTATEACTION(TPS_InCommentEnd),
1626 TPARSERSTATEACTION(TPS_InHostFirstDomain),
1627 TPARSERSTATEACTION(TPS_InHostDomainSecond),
1628 TPARSERSTATEACTION(TPS_InHostDomain),
1629 TPARSERSTATEACTION(TPS_InPortFirst),
1630 TPARSERSTATEACTION(TPS_InPort),
1631 TPARSERSTATEACTION(TPS_InHostFirstAN),
1632 TPARSERSTATEACTION(TPS_InHost),
1633 TPARSERSTATEACTION(TPS_InEmail),
1634 TPARSERSTATEACTION(TPS_InFileFirst),
1635 TPARSERSTATEACTION(TPS_InFileTwiddle),
1636 TPARSERSTATEACTION(TPS_InPathFirst),
1637 TPARSERSTATEACTION(TPS_InPathFirstFirst),
1638 TPARSERSTATEACTION(TPS_InPathSecond),
1639 TPARSERSTATEACTION(TPS_InFile),
1640 TPARSERSTATEACTION(TPS_InFileNext),
1641 TPARSERSTATEACTION(TPS_InURLPathFirst),
1642 TPARSERSTATEACTION(TPS_InURLPathStart),
1643 TPARSERSTATEACTION(TPS_InURLPath),
1644 TPARSERSTATEACTION(TPS_InFURL),
1645 TPARSERSTATEACTION(TPS_InProtocolFirst),
1646 TPARSERSTATEACTION(TPS_InProtocolSecond),
1647 TPARSERSTATEACTION(TPS_InProtocolEnd),
1648 TPARSERSTATEACTION(TPS_InHyphenAsciiWordFirst),
1649 TPARSERSTATEACTION(TPS_InHyphenAsciiWord),
1650 TPARSERSTATEACTION(TPS_InHyphenWordFirst),
1651 TPARSERSTATEACTION(TPS_InHyphenWord),
1652 TPARSERSTATEACTION(TPS_InHyphenNumWordFirst),
1653 TPARSERSTATEACTION(TPS_InHyphenNumWord),
1654 TPARSERSTATEACTION(TPS_InHyphenDigitLookahead),
1655 TPARSERSTATEACTION(TPS_InParseHyphen),
1656 TPARSERSTATEACTION(TPS_InParseHyphenHyphen),
1657 TPARSERSTATEACTION(TPS_InHyphenWordPart),
1658 TPARSERSTATEACTION(TPS_InHyphenAsciiWordPart),
1659 TPARSERSTATEACTION(TPS_InHyphenNumWordPart),
1660 TPARSERSTATEACTION(TPS_InHyphenUnsignedInt)
1661};
1662
1663
1664static bool
1665TParserGet(TParser *prs)
1666{
1667 const TParserStateActionItem *item = NULL;
1668
1669 CHECK_FOR_INTERRUPTS();
1670
1671 Assert(prs->state);
1672
1673 if (prs->state->posbyte >= prs->lenstr)
1674 return false;
1675
1676 prs->token = prs->str + prs->state->posbyte;
1677 prs->state->pushedAtAction = NULL;
1678
1679 /* look at string */
1680 while (prs->state->posbyte <= prs->lenstr)
1681 {
1682 if (prs->state->posbyte == prs->lenstr)
1683 prs->state->charlen = 0;
1684 else
1685 prs->state->charlen = (prs->charmaxlen == 1) ? prs->charmaxlen :
1686 pg_mblen(prs->str + prs->state->posbyte);
1687
1688 Assert(prs->state->posbyte + prs->state->charlen <= prs->lenstr);
1689 Assert(prs->state->state >= TPS_Base && prs->state->state < TPS_Null);
1690 Assert(Actions[prs->state->state].state == prs->state->state);
1691
1692 if (prs->state->pushedAtAction)
1693 {
1694 /* After a POP, pick up at the next test */
1695 item = prs->state->pushedAtAction + 1;
1696 prs->state->pushedAtAction = NULL;
1697 }
1698 else
1699 {
1700 item = Actions[prs->state->state].action;
1701 Assert(item != NULL);
1702 }
1703
1704 /* find action by character class */
1705 while (item->isclass)
1706 {
1707 prs->c = item->c;
1708 if (item->isclass(prs) != 0)
1709 break;
1710 item++;
1711 }
1712
1713#ifdef WPARSER_TRACE
1714 {
1715 TParserPosition *ptr;
1716
1717 fprintf(stderr, "state ");
1718 /* indent according to stack depth */
1719 for (ptr = prs->state->prev; ptr; ptr = ptr->prev)
1720 fprintf(stderr, " ");
1721 fprintf(stderr, "%s ", Actions[prs->state->state].state_name);
1722 if (prs->state->posbyte < prs->lenstr)
1723 fprintf(stderr, "at %c", *(prs->str + prs->state->posbyte));
1724 else
1725 fprintf(stderr, "at EOF");
1726 fprintf(stderr, " matched rule %d flags%s%s%s%s%s%s%s%s%s%s%s\n",
1727 (int) (item - Actions[prs->state->state].action),
1728 (item->flags & A_BINGO) ? " BINGO" : "",
1729 (item->flags & A_POP) ? " POP" : "",
1730 (item->flags & A_PUSH) ? " PUSH" : "",
1731 (item->flags & A_RERUN) ? " RERUN" : "",
1732 (item->flags & A_CLEAR) ? " CLEAR" : "",
1733 (item->flags & A_MERGE) ? " MERGE" : "",
1734 (item->flags & A_CLRALL) ? " CLRALL" : "",
1735 (item->tostate != TPS_Null) ? " tostate " : "",
1736 (item->tostate != TPS_Null) ? Actions[item->tostate].state_name : "",
1737 (item->type > 0) ? " type " : "",
1738 tok_alias[item->type]);
1739 }
1740#endif
1741
1742 /* call special handler if exists */
1743 if (item->special)
1744 item->special(prs);
1745
1746 /* BINGO, token is found */
1747 if (item->flags & A_BINGO)
1748 {
1749 Assert(item->type > 0);
1750 prs->lenbytetoken = prs->state->lenbytetoken;
1751 prs->lenchartoken = prs->state->lenchartoken;
1752 prs->state->lenbytetoken = prs->state->lenchartoken = 0;
1753 prs->type = item->type;
1754 }
1755
1756 /* do various actions by flags */
1757 if (item->flags & A_POP)
1758 { /* pop stored state in stack */
1759 TParserPosition *ptr = prs->state->prev;
1760
1761 pfree(prs->state);
1762 prs->state = ptr;
1763 Assert(prs->state);
1764 }
1765 else if (item->flags & A_PUSH)
1766 { /* push (store) state in stack */
1767 prs->state->pushedAtAction = item; /* remember where we push */
1768 prs->state = newTParserPosition(prs->state);
1769 }
1770 else if (item->flags & A_CLEAR)
1771 { /* clear previous pushed state */
1772 TParserPosition *ptr;
1773
1774 Assert(prs->state->prev);
1775 ptr = prs->state->prev->prev;
1776 pfree(prs->state->prev);
1777 prs->state->prev = ptr;
1778 }
1779 else if (item->flags & A_CLRALL)
1780 { /* clear all previous pushed state */
1781 TParserPosition *ptr;
1782
1783 while (prs->state->prev)
1784 {
1785 ptr = prs->state->prev->prev;
1786 pfree(prs->state->prev);
1787 prs->state->prev = ptr;
1788 }
1789 }
1790 else if (item->flags & A_MERGE)
1791 { /* merge posinfo with current and pushed state */
1792 TParserPosition *ptr = prs->state;
1793
1794 Assert(prs->state->prev);
1795 prs->state = prs->state->prev;
1796
1797 prs->state->posbyte = ptr->posbyte;
1798 prs->state->poschar = ptr->poschar;
1799 prs->state->charlen = ptr->charlen;
1800 prs->state->lenbytetoken = ptr->lenbytetoken;
1801 prs->state->lenchartoken = ptr->lenchartoken;
1802 pfree(ptr);
1803 }
1804
1805 /* set new state if pointed */
1806 if (item->tostate != TPS_Null)
1807 prs->state->state = item->tostate;
1808
1809 /* check for go away */
1810 if ((item->flags & A_BINGO) ||
1811 (prs->state->posbyte >= prs->lenstr &&
1812 (item->flags & A_RERUN) == 0))
1813 break;
1814
1815 /* go to beginning of loop if we should rerun or we just restore state */
1816 if (item->flags & (A_RERUN | A_POP))
1817 continue;
1818
1819 /* move forward */
1820 if (prs->state->charlen)
1821 {
1822 prs->state->posbyte += prs->state->charlen;
1823 prs->state->lenbytetoken += prs->state->charlen;
1824 prs->state->poschar++;
1825 prs->state->lenchartoken++;
1826 }
1827 }
1828
1829 return (item && (item->flags & A_BINGO));
1830}
1831
1832Datum
1833prsd_lextype(PG_FUNCTION_ARGS)
1834{
1835 LexDescr *descr = (LexDescr *) palloc(sizeof(LexDescr) * (LASTNUM + 1));
1836 int i;
1837
1838 for (i = 1; i <= LASTNUM; i++)
1839 {
1840 descr[i - 1].lexid = i;
1841 descr[i - 1].alias = pstrdup(tok_alias[i]);
1842 descr[i - 1].descr = pstrdup(lex_descr[i]);
1843 }
1844
1845 descr[LASTNUM].lexid = 0;
1846
1847 PG_RETURN_POINTER(descr);
1848}
1849
1850Datum
1851prsd_start(PG_FUNCTION_ARGS)
1852{
1853 PG_RETURN_POINTER(TParserInit((char *) PG_GETARG_POINTER(0), PG_GETARG_INT32(1)));
1854}
1855
1856Datum
1857prsd_nexttoken(PG_FUNCTION_ARGS)
1858{
1859 TParser *p = (TParser *) PG_GETARG_POINTER(0);
1860 char **t = (char **) PG_GETARG_POINTER(1);
1861 int *tlen = (int *) PG_GETARG_POINTER(2);
1862
1863 if (!TParserGet(p))
1864 PG_RETURN_INT32(0);
1865
1866 *t = p->token;
1867 *tlen = p->lenbytetoken;
1868
1869 PG_RETURN_INT32(p->type);
1870}
1871
1872Datum
1873prsd_end(PG_FUNCTION_ARGS)
1874{
1875 TParser *p = (TParser *) PG_GETARG_POINTER(0);
1876
1877 TParserClose(p);
1878 PG_RETURN_VOID();
1879}
1880
1881
1882/*
1883 * ts_headline support begins here
1884 */
1885
1886/* token type classification macros */
1887#define TS_IDIGNORE(x) ( (x)==TAG_T || (x)==PROTOCOL || (x)==SPACE || (x)==XMLENTITY )
1888#define HLIDREPLACE(x) ( (x)==TAG_T )
1889#define HLIDSKIP(x) ( (x)==URL_T || (x)==NUMHWORD || (x)==ASCIIHWORD || (x)==HWORD )
1890#define XMLHLIDSKIP(x) ( (x)==URL_T || (x)==NUMHWORD || (x)==ASCIIHWORD || (x)==HWORD )
1891#define NONWORDTOKEN(x) ( (x)==SPACE || HLIDREPLACE(x) || HLIDSKIP(x) )
1892#define NOENDTOKEN(x) ( NONWORDTOKEN(x) || (x)==SCIENTIFIC || (x)==VERSIONNUMBER || (x)==DECIMAL_T || (x)==SIGNEDINT || (x)==UNSIGNEDINT || TS_IDIGNORE(x) )
1893
1894/*
1895 * Macros useful in headline selection. These rely on availability of
1896 * "HeadlineParsedText *prs" describing some text, and "int shortword"
1897 * describing the "short word" length parameter.
1898 */
1899
1900/* Interesting words are non-repeated search terms */
1901#define INTERESTINGWORD(j) \
1902 (prs->words[j].item && !prs->words[j].repeated)
1903
1904/* Don't want to end at a non-word or a short word, unless interesting */
1905#define BADENDPOINT(j) \
1906 ((NOENDTOKEN(prs->words[j].type) || prs->words[j].len <= shortword) && \
1907 !INTERESTINGWORD(j))
1908
1909typedef struct
1910{
1911 /* one cover (well, really one fragment) for mark_hl_fragments */
1912 int32 startpos; /* fragment's starting word index */
1913 int32 endpos; /* ending word index (inclusive) */
1914 int32 poslen; /* number of interesting words */
1915 int32 curlen; /* total number of words */
1916 bool chosen; /* chosen? */
1917 bool excluded; /* excluded? */
1918} CoverPos;
1919
1920typedef struct
1921{
1922 /* callback data for checkcondition_HL */
1923 HeadlineWordEntry *words;
1924 int len;
1925} hlCheck;
1926
1927
1928/*
1929 * TS_execute callback for matching a tsquery operand to headline words
1930 *
1931 * Note: it's tempting to report words[] indexes as pos values to save
1932 * searching in hlCover; but that would screw up phrase matching, which
1933 * expects to measure distances in lexemes not tokens.
1934 */
1935static TSTernaryValue
1936checkcondition_HL(void *opaque, QueryOperand *val, ExecPhraseData *data)
1937{
1938 hlCheck *checkval = (hlCheck *) opaque;
1939 int i;
1940
1941 /* scan words array for matching items */
1942 for (i = 0; i < checkval->len; i++)
1943 {
1944 if (checkval->words[i].item == val)
1945 {
1946 /* if data == NULL, don't need to report positions */
1947 if (!data)
1948 return TS_YES;
1949
1950 if (!data->pos)
1951 {
1952 data->pos = palloc(sizeof(WordEntryPos) * checkval->len);
1953 data->allocated = true;
1954 data->npos = 1;
1955 data->pos[0] = checkval->words[i].pos;
1956 }
1957 else if (data->pos[data->npos - 1] < checkval->words[i].pos)
1958 {
1959 data->pos[data->npos++] = checkval->words[i].pos;
1960 }
1961 }
1962 }
1963
1964 if (data && data->npos > 0)
1965 return TS_YES;
1966
1967 return TS_NO;
1968}
1969
1970/*
1971 * hlCover: try to find a substring of prs' word list that satisfies query
1972 *
1973 * locations is the result of TS_execute_locations() for the query.
1974 * We use this to identify plausible subranges of the query.
1975 *
1976 * *nextpos is the lexeme position (NOT word index) to start the search
1977 * at. Caller should initialize this to zero. If successful, we'll
1978 * advance it to the next place to search at.
1979 *
1980 * On success, sets *p to first word index and *q to last word index of the
1981 * cover substring, and returns true.
1982 *
1983 * The result is a minimal cover, in the sense that both *p and *q will be
1984 * words used in the query.
1985 */
1986static bool
1987hlCover(HeadlineParsedText *prs, TSQuery query, List *locations,
1988 int *nextpos, int *p, int *q)
1989{
1990 int pos = *nextpos;
1991
1992 /* This loop repeats when our selected word-range fails the query */
1993 for (;;)
1994 {
1995 int posb,
1996 pose;
1997 ListCell *lc;
1998
1999 /*
2000 * For each AND'ed query term or phrase, find its first occurrence at
2001 * or after pos; set pose to the maximum of those positions.
2002 *
2003 * We need not consider ORs or NOTs here; see the comments for
2004 * TS_execute_locations(). Rechecking the match with TS_execute(),
2005 * below, will deal with any ensuing imprecision.
2006 */
2007 pose = -1;
2008 foreach(lc, locations)
2009 {
2010 ExecPhraseData *pdata = (ExecPhraseData *) lfirst(lc);
2011 int first = -1;
2012
2013 for (int i = 0; i < pdata->npos; i++)
2014 {
2015 /* For phrase matches, use the ending lexeme */
2016 int endp = pdata->pos[i];
2017
2018 if (endp >= pos)
2019 {
2020 first = endp;
2021 break;
2022 }
2023 }
2024 if (first < 0)
2025 return false; /* no more matches for this term */
2026 if (first > pose)
2027 pose = first;
2028 }
2029
2030 if (pose < 0)
2031 return false; /* we only get here if empty list */
2032
2033 /*
2034 * Now, for each AND'ed query term or phrase, find its last occurrence
2035 * at or before pose; set posb to the minimum of those positions.
2036 *
2037 * We start posb at INT_MAX - 1 to guarantee no overflow if we compute
2038 * posb + 1 below.
2039 */
2040 posb = INT_MAX - 1;
2041 foreach(lc, locations)
2042 {
2043 ExecPhraseData *pdata = (ExecPhraseData *) lfirst(lc);
2044 int last = -1;
2045
2046 for (int i = pdata->npos - 1; i >= 0; i--)
2047 {
2048 /* For phrase matches, use the starting lexeme */
2049 int startp = pdata->pos[i] - pdata->width;
2050
2051 if (startp <= pose)
2052 {
2053 last = startp;
2054 break;
2055 }
2056 }
2057 if (last < posb)
2058 posb = last;
2059 }
2060
2061 /*
2062 * We could end up with posb to the left of pos, in case some phrase
2063 * match crosses pos. Try the match starting at pos anyway, since the
2064 * result of TS_execute_locations is imprecise for phrase matches OR'd
2065 * with plain matches; that is, if the query is "(A <-> B) | C" then C
2066 * could match at pos even though the phrase match would have to
2067 * extend to the left of pos.
2068 */
2069 posb = Max(posb, pos);
2070
2071 /* This test probably always succeeds, but be paranoid */
2072 if (posb <= pose)
2073 {
2074 /*
2075 * posb .. pose is now the shortest, earliest-after-pos range of
2076 * lexeme positions containing all the query terms. It will
2077 * contain all phrase matches, too, except in the corner case
2078 * described just above.
2079 *
2080 * Now convert these lexeme positions to indexes in prs->words[].
2081 */
2082 int idxb = -1;
2083 int idxe = -1;
2084
2085 for (int i = 0; i < prs->curwords; i++)
2086 {
2087 if (prs->words[i].item == NULL)
2088 continue;
2089 if (idxb < 0 && prs->words[i].pos >= posb)
2090 idxb = i;
2091 if (prs->words[i].pos <= pose)
2092 idxe = i;
2093 else
2094 break;
2095 }
2096
2097 /* This test probably always succeeds, but be paranoid */
2098 if (idxb >= 0 && idxe >= idxb)
2099 {
2100 /*
2101 * Finally, check that the selected range satisfies the query.
2102 * This should succeed in all simple cases; but odd cases
2103 * involving non-top-level NOT conditions or phrase matches
2104 * OR'd with other things could fail, since the result of
2105 * TS_execute_locations doesn't fully represent such things.
2106 */
2107 hlCheck ch;
2108
2109 ch.words = &(prs->words[idxb]);
2110 ch.len = idxe - idxb + 1;
2111 if (TS_execute(GETQUERY(query), &ch,
2112 TS_EXEC_EMPTY, checkcondition_HL))
2113 {
2114 /* Match! Advance *nextpos and return the word range. */
2115 *nextpos = posb + 1;
2116 *p = idxb;
2117 *q = idxe;
2118 return true;
2119 }
2120 }
2121 }
2122
2123 /*
2124 * Advance pos and try again. Any later workable match must start
2125 * beyond posb.
2126 */
2127 pos = posb + 1;
2128 }
2129 /* Can't get here, but stupider compilers complain if we leave it off */
2130 return false;
2131}
2132
2133/*
2134 * Apply suitable highlight marking to words selected by headline selector
2135 *
2136 * The words from startpos to endpos inclusive are marked per highlightall
2137 */
2138static void
2139mark_fragment(HeadlineParsedText *prs, bool highlightall,
2140 int startpos, int endpos)
2141{
2142 int i;
2143
2144 for (i = startpos; i <= endpos; i++)
2145 {
2146 if (prs->words[i].item)
2147 prs->words[i].selected = 1;
2148 if (!highlightall)
2149 {
2150 if (HLIDREPLACE(prs->words[i].type))
2151 prs->words[i].replace = 1;
2152 else if (HLIDSKIP(prs->words[i].type))
2153 prs->words[i].skip = 1;
2154 }
2155 else
2156 {
2157 if (XMLHLIDSKIP(prs->words[i].type))
2158 prs->words[i].skip = 1;
2159 }
2160
2161 prs->words[i].in = (prs->words[i].repeated) ? 0 : 1;
2162 }
2163}
2164
2165/*
2166 * split a cover substring into fragments not longer than max_words
2167 *
2168 * At entry, *startpos and *endpos are the (remaining) bounds of the cover
2169 * substring. They are updated to hold the bounds of the next fragment.
2170 *
2171 * *curlen and *poslen are set to the fragment's length, in words and
2172 * interesting words respectively.
2173 */
2174static void
2175get_next_fragment(HeadlineParsedText *prs, int *startpos, int *endpos,
2176 int *curlen, int *poslen, int max_words)
2177{
2178 int i;
2179
2180 /*
2181 * Objective: select a fragment of words between startpos and endpos such
2182 * that it has at most max_words and both ends have query words. If the
2183 * startpos and endpos are the endpoints of the cover and the cover has
2184 * fewer words than max_words, then this function should just return the
2185 * cover
2186 */
2187 /* first move startpos to an item */
2188 for (i = *startpos; i <= *endpos; i++)
2189 {
2190 *startpos = i;
2191 if (INTERESTINGWORD(i))
2192 break;
2193 }
2194 /* cut endpos to have only max_words */
2195 *curlen = 0;
2196 *poslen = 0;
2197 for (i = *startpos; i <= *endpos && *curlen < max_words; i++)
2198 {
2199 if (!NONWORDTOKEN(prs->words[i].type))
2200 *curlen += 1;
2201 if (INTERESTINGWORD(i))
2202 *poslen += 1;
2203 }
2204 /* if the cover was cut then move back endpos to a query item */
2205 if (*endpos > i)
2206 {
2207 *endpos = i;
2208 for (i = *endpos; i >= *startpos; i--)
2209 {
2210 *endpos = i;
2211 if (INTERESTINGWORD(i))
2212 break;
2213 if (!NONWORDTOKEN(prs->words[i].type))
2214 *curlen -= 1;
2215 }
2216 }
2217}
2218
2219/*
2220 * Headline selector used when MaxFragments > 0
2221 *
2222 * Note: in this mode, highlightall is disregarded for phrase selection;
2223 * it only controls presentation details.
2224 */
2225static void
2226mark_hl_fragments(HeadlineParsedText *prs, TSQuery query, List *locations,
2227 bool highlightall,
2228 int shortword, int min_words,
2229 int max_words, int max_fragments)
2230{
2231 int32 poslen,
2232 curlen,
2233 i,
2234 f,
2235 num_f = 0;
2236 int32 stretch,
2237 maxstretch,
2238 posmarker;
2239
2240 int32 startpos = 0,
2241 endpos = 0,
2242 nextpos = 0,
2243 p = 0,
2244 q = 0;
2245
2246 int32 numcovers = 0,
2247 maxcovers = 32;
2248
2249 int32 minI,
2250 minwords,
2251 maxitems;
2252 CoverPos *covers;
2253
2254 covers = palloc(maxcovers * sizeof(CoverPos));
2255
2256 /* get all covers */
2257 while (hlCover(prs, query, locations, &nextpos, &p, &q))
2258 {
2259 startpos = p;
2260 endpos = q;
2261
2262 /*
2263 * Break the cover into smaller fragments such that each fragment has
2264 * at most max_words. Also ensure that each end of each fragment is a
2265 * query word. This will allow us to stretch the fragment in either
2266 * direction
2267 */
2268
2269 while (startpos <= endpos)
2270 {
2271 get_next_fragment(prs, &startpos, &endpos, &curlen, &poslen, max_words);
2272 if (numcovers >= maxcovers)
2273 {
2274 maxcovers *= 2;
2275 covers = repalloc(covers, sizeof(CoverPos) * maxcovers);
2276 }
2277 covers[numcovers].startpos = startpos;
2278 covers[numcovers].endpos = endpos;
2279 covers[numcovers].curlen = curlen;
2280 covers[numcovers].poslen = poslen;
2281 covers[numcovers].chosen = false;
2282 covers[numcovers].excluded = false;
2283 numcovers++;
2284 startpos = endpos + 1;
2285 endpos = q;
2286 }
2287 }
2288
2289 /* choose best covers */
2290 for (f = 0; f < max_fragments; f++)
2291 {
2292 maxitems = 0;
2293 minwords = PG_INT32_MAX;
2294 minI = -1;
2295
2296 /*
2297 * Choose the cover that contains max items. In case of tie choose the
2298 * one with smaller number of words.
2299 */
2300 for (i = 0; i < numcovers; i++)
2301 {
2302 if (!covers[i].chosen && !covers[i].excluded &&
2303 (maxitems < covers[i].poslen ||
2304 (maxitems == covers[i].poslen &&
2305 minwords > covers[i].curlen)))
2306 {
2307 maxitems = covers[i].poslen;
2308 minwords = covers[i].curlen;
2309 minI = i;
2310 }
2311 }
2312 /* if a cover was found mark it */
2313 if (minI >= 0)
2314 {
2315 covers[minI].chosen = true;
2316 /* adjust the size of cover */
2317 startpos = covers[minI].startpos;
2318 endpos = covers[minI].endpos;
2319 curlen = covers[minI].curlen;
2320 /* stretch the cover if cover size is lower than max_words */
2321 if (curlen < max_words)
2322 {
2323 /* divide the stretch on both sides of cover */
2324 maxstretch = (max_words - curlen) / 2;
2325
2326 /*
2327 * first stretch the startpos stop stretching if 1. we hit the
2328 * beginning of document 2. exceed maxstretch 3. we hit an
2329 * already marked fragment
2330 */
2331 stretch = 0;
2332 posmarker = startpos;
2333 for (i = startpos - 1; i >= 0 && stretch < maxstretch && !prs->words[i].in; i--)
2334 {
2335 if (!NONWORDTOKEN(prs->words[i].type))
2336 {
2337 curlen++;
2338 stretch++;
2339 }
2340 posmarker = i;
2341 }
2342 /* cut back startpos till we find a good endpoint */
2343 for (i = posmarker; i < startpos && BADENDPOINT(i); i++)
2344 {
2345 if (!NONWORDTOKEN(prs->words[i].type))
2346 curlen--;
2347 }
2348 startpos = i;
2349 /* now stretch the endpos as much as possible */
2350 posmarker = endpos;
2351 for (i = endpos + 1; i < prs->curwords && curlen < max_words && !prs->words[i].in; i++)
2352 {
2353 if (!NONWORDTOKEN(prs->words[i].type))
2354 curlen++;
2355 posmarker = i;
2356 }
2357 /* cut back endpos till we find a good endpoint */
2358 for (i = posmarker; i > endpos && BADENDPOINT(i); i--)
2359 {
2360 if (!NONWORDTOKEN(prs->words[i].type))
2361 curlen--;
2362 }
2363 endpos = i;
2364 }
2365 covers[minI].startpos = startpos;
2366 covers[minI].endpos = endpos;
2367 covers[minI].curlen = curlen;
2368 /* Mark the chosen fragments (covers) */
2369 mark_fragment(prs, highlightall, startpos, endpos);
2370 num_f++;
2371 /* Exclude covers overlapping this one from future consideration */
2372 for (i = 0; i < numcovers; i++)
2373 {
2374 if (i != minI &&
2375 ((covers[i].startpos >= startpos &&
2376 covers[i].startpos <= endpos) ||
2377 (covers[i].endpos >= startpos &&
2378 covers[i].endpos <= endpos) ||
2379 (covers[i].startpos < startpos &&
2380 covers[i].endpos > endpos)))
2381 covers[i].excluded = true;
2382 }
2383 }
2384 else
2385 break; /* no selectable covers remain */
2386 }
2387
2388 /* show the first min_words words if we have not marked anything */
2389 if (num_f <= 0)
2390 {
2391 startpos = curlen = 0;
2392 endpos = -1;
2393 for (i = 0; i < prs->curwords && curlen < min_words; i++)
2394 {
2395 if (!NONWORDTOKEN(prs->words[i].type))
2396 curlen++;
2397 endpos = i;
2398 }
2399 mark_fragment(prs, highlightall, startpos, endpos);
2400 }
2401
2402 pfree(covers);
2403}
2404
2405/*
2406 * Headline selector used when MaxFragments == 0
2407 */
2408static void
2409mark_hl_words(HeadlineParsedText *prs, TSQuery query, List *locations,
2410 bool highlightall,
2411 int shortword, int min_words, int max_words)
2412{
2413 int nextpos = 0,
2414 p = 0,
2415 q = 0;
2416 int bestb = -1,
2417 beste = -1;
2418 int bestlen = -1;
2419 bool bestcover = false;
2420 int pose,
2421 posb,
2422 poslen,
2423 curlen;
2424 bool poscover;
2425 int i;
2426
2427 if (!highlightall)
2428 {
2429 /* examine all covers, select a headline using the best one */
2430 while (hlCover(prs, query, locations, &nextpos, &p, &q))
2431 {
2432 /*
2433 * Count words (curlen) and interesting words (poslen) within
2434 * cover, but stop once we reach max_words. This step doesn't
2435 * consider whether that's a good stopping point. posb and pose
2436 * are set to the start and end indexes of the possible headline.
2437 */
2438 curlen = 0;
2439 poslen = 0;
2440 posb = pose = p;
2441 for (i = p; i <= q && curlen < max_words; i++)
2442 {
2443 if (!NONWORDTOKEN(prs->words[i].type))
2444 curlen++;
2445 if (INTERESTINGWORD(i))
2446 poslen++;
2447 pose = i;
2448 }
2449
2450 if (curlen < max_words)
2451 {
2452 /*
2453 * We have room to lengthen the headline, so search forward
2454 * until it's full or we find a good stopping point. We'll
2455 * reconsider the word at "q", then move forward.
2456 */
2457 for (i = i - 1; i < prs->curwords && curlen < max_words; i++)
2458 {
2459 if (i > q)
2460 {
2461 if (!NONWORDTOKEN(prs->words[i].type))
2462 curlen++;
2463 if (INTERESTINGWORD(i))
2464 poslen++;
2465 }
2466 pose = i;
2467 if (BADENDPOINT(i))
2468 continue;
2469 if (curlen >= min_words)
2470 break;
2471 }
2472 if (curlen < min_words)
2473 {
2474 /*
2475 * Reached end of text and our headline is still shorter
2476 * than min_words, so try to extend it to the left.
2477 */
2478 for (i = p - 1; i >= 0; i--)
2479 {
2480 if (!NONWORDTOKEN(prs->words[i].type))
2481 curlen++;
2482 if (INTERESTINGWORD(i))
2483 poslen++;
2484 if (curlen >= max_words)
2485 break;
2486 if (BADENDPOINT(i))
2487 continue;
2488 if (curlen >= min_words)
2489 break;
2490 }
2491 posb = (i >= 0) ? i : 0;
2492 }
2493 }
2494 else
2495 {
2496 /*
2497 * Can't make headline longer, so consider making it shorter
2498 * if needed to avoid a bad endpoint.
2499 */
2500 if (i > q)
2501 i = q;
2502 for (; curlen > min_words; i--)
2503 {
2504 if (!BADENDPOINT(i))
2505 break;
2506 if (!NONWORDTOKEN(prs->words[i].type))
2507 curlen--;
2508 if (INTERESTINGWORD(i))
2509 poslen--;
2510 pose = i - 1;
2511 }
2512 }
2513
2514 /*
2515 * Check whether the proposed headline includes the original
2516 * cover; it might not if we trimmed it due to max_words.
2517 */
2518 poscover = (posb <= p && pose >= q);
2519
2520 /*
2521 * Adopt this headline if it's better than the last one, giving
2522 * highest priority to headlines including the cover, then to
2523 * headlines with more interesting words, then to headlines with
2524 * good stopping points. (Since bestlen is initially -1, we will
2525 * certainly adopt the first headline.)
2526 */
2527 if (poscover > bestcover ||
2528 (poscover == bestcover && poslen > bestlen) ||
2529 (poscover == bestcover && poslen == bestlen &&
2530 !BADENDPOINT(pose) && BADENDPOINT(beste)))
2531 {
2532 bestb = posb;
2533 beste = pose;
2534 bestlen = poslen;
2535 bestcover = poscover;
2536 }
2537 }
2538
2539 /*
2540 * If we found nothing acceptable, select min_words words starting at
2541 * the beginning.
2542 */
2543 if (bestlen < 0)
2544 {
2545 curlen = 0;
2546 pose = -1;
2547 for (i = 0; i < prs->curwords && curlen < min_words; i++)
2548 {
2549 if (!NONWORDTOKEN(prs->words[i].type))
2550 curlen++;
2551 pose = i;
2552 }
2553 bestb = 0;
2554 beste = pose;
2555 }
2556 }
2557 else
2558 {
2559 /* highlightall mode: headline is whole document */
2560 bestb = 0;
2561 beste = prs->curwords - 1;
2562 }
2563
2564 mark_fragment(prs, highlightall, bestb, beste);
2565}
2566
2567/*
2568 * Default parser's prsheadline function
2569 */
2570Datum
2571prsd_headline(PG_FUNCTION_ARGS)
2572{
2573 HeadlineParsedText *prs = (HeadlineParsedText *) PG_GETARG_POINTER(0);
2574 List *prsoptions = (List *) PG_GETARG_POINTER(1);
2575 TSQuery query = PG_GETARG_TSQUERY(2);
2576 List *locations;
2577
2578 /* default option values: */
2579 int min_words = 15;
2580 int max_words = 35;
2581 int shortword = 3;
2582 int max_fragments = 0;
2583 bool highlightall = false;
2584 ListCell *l;
2585
2586 /* Extract configuration option values */
2587 prs->startsel = NULL;
2588 prs->stopsel = NULL;
2589 prs->fragdelim = NULL;
2590 foreach(l, prsoptions)
2591 {
2592 DefElem *defel = (DefElem *) lfirst(l);
2593 char *val = defGetString(defel);
2594
2595 if (pg_strcasecmp(defel->defname, "MaxWords") == 0)
2596 max_words = pg_strtoint32(val);
2597 else if (pg_strcasecmp(defel->defname, "MinWords") == 0)
2598 min_words = pg_strtoint32(val);
2599 else if (pg_strcasecmp(defel->defname, "ShortWord") == 0)
2600 shortword = pg_strtoint32(val);
2601 else if (pg_strcasecmp(defel->defname, "MaxFragments") == 0)
2602 max_fragments = pg_strtoint32(val);
2603 else if (pg_strcasecmp(defel->defname, "StartSel") == 0)
2604 prs->startsel = pstrdup(val);
2605 else if (pg_strcasecmp(defel->defname, "StopSel") == 0)
2606 prs->stopsel = pstrdup(val);
2607 else if (pg_strcasecmp(defel->defname, "FragmentDelimiter") == 0)
2608 prs->fragdelim = pstrdup(val);
2609 else if (pg_strcasecmp(defel->defname, "HighlightAll") == 0)
2610 highlightall = (pg_strcasecmp(val, "1") == 0 ||
2611 pg_strcasecmp(val, "on") == 0 ||
2612 pg_strcasecmp(val, "true") == 0 ||
2613 pg_strcasecmp(val, "t") == 0 ||
2614 pg_strcasecmp(val, "y") == 0 ||
2615 pg_strcasecmp(val, "yes") == 0);
2616 else
2617 ereport(ERROR,
2618 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2619 errmsg("unrecognized headline parameter: \"%s\"",
2620 defel->defname)));
2621 }
2622
2623 /* in HighlightAll mode these parameters are ignored */
2624 if (!highlightall)
2625 {
2626 if (min_words >= max_words)
2627 ereport(ERROR,
2628 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2629 errmsg("%s must be less than %s", "MinWords", "MaxWords")));
2630 if (min_words <= 0)
2631 ereport(ERROR,
2632 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2633 errmsg("%s must be positive", "MinWords")));
2634 if (shortword < 0)
2635 ereport(ERROR,
2636 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2637 errmsg("%s must be >= 0", "ShortWord")));
2638 if (max_fragments < 0)
2639 ereport(ERROR,
2640 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2641 errmsg("%s must be >= 0", "MaxFragments")));
2642 }
2643
2644 /* Locate words and phrases matching the query */
2645 if (query->size > 0)
2646 {
2647 hlCheck ch;
2648
2649 ch.words = prs->words;
2650 ch.len = prs->curwords;
2651 locations = TS_execute_locations(GETQUERY(query), &ch, TS_EXEC_EMPTY,
2652 checkcondition_HL);
2653 }
2654 else
2655 locations = NIL; /* empty query matches nothing */
2656
2657 /* Apply appropriate headline selector */
2658 if (max_fragments == 0)
2659 mark_hl_words(prs, query, locations, highlightall, shortword,
2660 min_words, max_words);
2661 else
2662 mark_hl_fragments(prs, query, locations, highlightall, shortword,
2663 min_words, max_words, max_fragments);
2664
2665 /* Fill in default values for string options */
2666 if (!prs->startsel)
2667 prs->startsel = pstrdup("<b>");
2668 if (!prs->stopsel)
2669 prs->stopsel = pstrdup("</b>");
2670 if (!prs->fragdelim)
2671 prs->fragdelim = pstrdup(" ... ");
2672
2673 /* Caller will need these lengths, too */
2674 prs->startsellen = strlen(prs->startsel);
2675 prs->stopsellen = strlen(prs->stopsel);
2676 prs->fragdelimlen = strlen(prs->fragdelim);
2677
2678 PG_RETURN_POINTER(prs);
2679}
GETQUERY
#define GETQUERY(x)
Definition: _int.h:157
print
void print(const void *obj)
Definition: print.c:36
PG_INT32_MAX
#define PG_INT32_MAX
Definition: c.h:599
Max
#define Max(x, y)
Definition: c.h:1002
int32
int32_t int32
Definition: c.h:539
uint16
uint16_t uint16
Definition: c.h:542
lengthof
#define lengthof(array)
Definition: c.h:792
fprintf
#define fprintf(file, fmt, msg)
Definition: cubescan.l:21
defGetString
char * defGetString(DefElem *def)
Definition: define.c:35
errcode
int errcode(int sqlerrcode)
Definition: elog.c:863
errmsg
int errmsg(const char *fmt,...)
Definition: elog.c:1080
ERROR
#define ERROR
Definition: elog.h:39
ereport
#define ereport(elevel,...)
Definition: elog.h:150
PG_RETURN_VOID
#define PG_RETURN_VOID()
Definition: fmgr.h:349
PG_GETARG_POINTER
#define PG_GETARG_POINTER(n)
Definition: fmgr.h:276
PG_RETURN_INT32
#define PG_RETURN_INT32(x)
Definition: fmgr.h:354
PG_GETARG_INT32
#define PG_GETARG_INT32(n)
Definition: fmgr.h:269
PG_RETURN_POINTER
#define PG_RETURN_POINTER(x)
Definition: fmgr.h:361
PG_FUNCTION_ARGS
#define PG_FUNCTION_ARGS
Definition: fmgr.h:193
Assert
Assert(PointerIsAligned(start, uint64))
str
const char * str
Definition: hashfn_unstable.h:254
val
long val
Definition: informix.c:689
i
int i
Definition: isn.c:77
pg_wchar
unsigned int pg_wchar
Definition: mbprint.c:31
GetDatabaseEncoding
int GetDatabaseEncoding(void)
Definition: mbutils.c:1262
pg_dsplen
int pg_dsplen(const char *mbstr)
Definition: mbutils.c:1031
pg_database_encoding_max_length
int pg_database_encoding_max_length(void)
Definition: mbutils.c:1547
pg_mb2wchar_with_len
int pg_mb2wchar_with_len(const char *from, pg_wchar *to, int len)
Definition: mbutils.c:987
pg_mblen
int pg_mblen(const char *mbstr)
Definition: mbutils.c:1024
pstrdup
char * pstrdup(const char *in)
Definition: mcxt.c:1759
repalloc
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1610
pfree
void pfree(void *pointer)
Definition: mcxt.c:1594
palloc0
void * palloc0(Size size)
Definition: mcxt.c:1395
palloc
void * palloc(Size size)
Definition: mcxt.c:1365
CHECK_FOR_INTERRUPTS
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:123
pg_strtoint32
int32 pg_strtoint32(const char *s)
Definition: numutils.c:383
lower
Datum lower(PG_FUNCTION_ARGS)
Definition: oracle_compat.c:49
upper
Datum upper(PG_FUNCTION_ARGS)
Definition: oracle_compat.c:80
len
const void size_t len
Definition: pg_crc32c_sse42.c:28
data
const void * data
Definition: pg_crc32c_sse42.c:27
lfirst
#define lfirst(lc)
Definition: pg_list.h:172
NIL
#define NIL
Definition: pg_list.h:68
pg_locale.h
endpos
static XLogRecPtr endpos
Definition: pg_receivewal.c:56
startpos
static XLogRecPtr startpos
Definition: pg_recvlogical.c:50
pg_wchar.h
PG_UTF8
@ PG_UTF8
Definition: pg_wchar.h:232
pg_strcasecmp
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
pg_strncasecmp
int pg_strncasecmp(const char *s1, const char *s2, size_t n)
Definition: pgstrcasecmp.c:69
Datum
uint64_t Datum
Definition: postgres.h:70
c
char * c
Definition: preproc-cursor.c:31
check_stack_depth
void check_stack_depth(void)
Definition: stack_depth.c:95
CoverPos::chosen
bool chosen
Definition: wparser_def.c:1916
CoverPos::endpos
int32 endpos
Definition: wparser_def.c:1913
CoverPos::curlen
int32 curlen
Definition: wparser_def.c:1915
CoverPos::startpos
int32 startpos
Definition: wparser_def.c:1912
CoverPos::excluded
bool excluded
Definition: wparser_def.c:1917
CoverPos::poslen
int32 poslen
Definition: wparser_def.c:1914
DefElem::defname
char * defname
Definition: parsenodes.h:843
ExecPhraseData::pos
WordEntryPos * pos
Definition: ts_utils.h:166
HeadlineParsedText::words
HeadlineWordEntry * words
Definition: ts_public.h:76
HeadlineWordEntry::pos
WordEntryPos pos
Definition: ts_public.h:68
HeadlineWordEntry::item
QueryOperand * item
Definition: ts_public.h:70
LexDescr::alias
char * alias
Definition: ts_public.h:28
LexDescr::lexid
int lexid
Definition: ts_public.h:27
LexDescr::descr
char * descr
Definition: ts_public.h:29
List
Definition: pg_list.h:54
TParserPosition::pushedAtAction
const TParserStateActionItem * pushedAtAction
Definition: wparser_def.c:238
TParserPosition::prev
struct TParserPosition * prev
Definition: wparser_def.c:237
TParserPosition::state
TParserState state
Definition: wparser_def.c:236
TParserStateActionItem::isclass
TParserCharTest isclass
Definition: wparser_def.c:211
TParserStateActionItem::tostate
TParserState tostate
Definition: wparser_def.c:214
TParserStateActionItem::special
TParserSpecial special
Definition: wparser_def.c:216
TParserStateAction::action
const TParserStateActionItem * action
Definition: wparser_def.c:1564
TParserStateAction::state
TParserState state
Definition: wparser_def.c:1565
TParser::str
char * str
Definition: wparser_def.c:244
TParser::pgwstr
pg_wchar * pgwstr
Definition: wparser_def.c:246
TParser::lenstr
int lenstr
Definition: wparser_def.c:245
TParser::token
char * token
Definition: wparser_def.c:258
TParser::type
int type
Definition: wparser_def.c:261
TParser::charmaxlen
int charmaxlen
Definition: wparser_def.c:249
TParser::wanthost
bool wanthost
Definition: wparser_def.c:252
TParser::lenbytetoken
int lenbytetoken
Definition: wparser_def.c:259
TParser::ignore
bool ignore
Definition: wparser_def.c:251
TParser::state
TParserPosition * state
Definition: wparser_def.c:250
TParser::lenchartoken
int lenchartoken
Definition: wparser_def.c:260
TParser::c
char c
Definition: wparser_def.c:255
TSQueryData::size
int32 size
Definition: ts_type.h:221
hlCheck::words
HeadlineWordEntry * words
Definition: wparser_def.c:1923
ts_public.h
ts_type.h
PG_GETARG_TSQUERY
#define PG_GETARG_TSQUERY(n)
Definition: ts_type.h:266
WordEntryPos
uint16 WordEntryPos
Definition: ts_type.h:63
ts_utils.h
TSTernaryValue
TSTernaryValue
Definition: ts_utils.h:133
TS_NO
@ TS_NO
Definition: ts_utils.h:134
TS_YES
@ TS_YES
Definition: ts_utils.h:135
TS_EXEC_EMPTY
#define TS_EXEC_EMPTY
Definition: ts_utils.h:188
TS_execute
bool TS_execute(QueryItem *curitem, void *arg, uint32 flags, TSExecuteCallback chkcond)
Definition: tsvector_op.c:1854
TS_execute_locations
List * TS_execute_locations(QueryItem *curitem, void *arg, uint32 flags, TSExecuteCallback chkcond)
Definition: tsvector_op.c:2007
actionTPS_InParseHyphen
static const TParserStateActionItem actionTPS_InParseHyphen[]
Definition: wparser_def.c:1510
actionTPS_InXMLEntityNumFirst
static const TParserStateActionItem actionTPS_InXMLEntityNumFirst[]
Definition: wparser_def.c:1121
actionTPS_InHyphenWordFirst
static const TParserStateActionItem actionTPS_InHyphenWordFirst[]
Definition: wparser_def.c:1471
NONWORDTOKEN
#define NONWORDTOKEN(x)
Definition: wparser_def.c:1891
actionTPS_InXMLEntityFirst
static const TParserStateActionItem actionTPS_InXMLEntityFirst[]
Definition: wparser_def.c:1101
actionTPS_InHostFirstAN
static const TParserStateActionItem actionTPS_InHostFirstAN[]
Definition: wparser_def.c:1328
VERSIONNUMBER
#define VERSIONNUMBER
Definition: wparser_def.c:43
actionTPS_InHyphenNumWordPart
static const TParserStateActionItem actionTPS_InHyphenNumWordPart[]
Definition: wparser_def.c:1543
BADENDPOINT
#define BADENDPOINT(j)
Definition: wparser_def.c:1905
ASCIIWORD
#define ASCIIWORD
Definition: wparser_def.c:36
PROTOCOL
#define PROTOCOL
Definition: wparser_def.c:49
actionTPS_InPathSecond
static const TParserStateActionItem actionTPS_InPathSecond[]
Definition: wparser_def.c:1388
actionTPS_InPathFirst
static const TParserStateActionItem actionTPS_InPathFirst[]
Definition: wparser_def.c:1371
actionTPS_InHostDomainSecond
static const TParserStateActionItem actionTPS_InHostDomainSecond[]
Definition: wparser_def.c:1288
actionTPS_InCloseCommentFirst
static const TParserStateActionItem actionTPS_InCloseCommentFirst[]
Definition: wparser_def.c:1264
SpecialFURL
static void SpecialFURL(TParser *prs)
Definition: wparser_def.c:546
actionTPS_InCommentEnd
static const TParserStateActionItem actionTPS_InCommentEnd[]
Definition: wparser_def.c:1277
TParser
struct TParser TParser
checkcondition_HL
static TSTernaryValue checkcondition_HL(void *opaque, QueryOperand *val, ExecPhraseData *data)
Definition: wparser_def.c:1936
_make_compiler_happy
void _make_compiler_happy(void)
Definition: wparser_def.c:495
actionTPS_InURLPathStart
static const TParserStateActionItem actionTPS_InURLPathStart[]
Definition: wparser_def.c:1421
actionTPS_InHostFirstDomain
static const TParserStateActionItem actionTPS_InHostFirstDomain[]
Definition: wparser_def.c:1281
actionTPS_InHyphenDigitLookahead
static const TParserStateActionItem actionTPS_InHyphenDigitLookahead[]
Definition: wparser_def.c:1502
actionTPS_InHostDomain
static const TParserStateActionItem actionTPS_InHostDomain[]
Definition: wparser_def.c:1299
actionTPS_InVersion
static const TParserStateActionItem actionTPS_InVersion[]
Definition: wparser_def.c:1074
XMLHLIDSKIP
#define XMLHLIDSKIP(x)
Definition: wparser_def.c:1890
actionTPS_InHyphenAsciiWordFirst
static const TParserStateActionItem actionTPS_InHyphenAsciiWordFirst[]
Definition: wparser_def.c:1453
prsd_nexttoken
Datum prsd_nexttoken(PG_FUNCTION_ARGS)
Definition: wparser_def.c:1857
actionTPS_InTagName
static const TParserStateActionItem actionTPS_InTagName[]
Definition: wparser_def.c:1178
DECIMAL_T
#define DECIMAL_T
Definition: wparser_def.c:55
actionTPS_InFileNext
static const TParserStateActionItem actionTPS_InFileNext[]
Definition: wparser_def.c:1407
actionTPS_InXMLEntity
static const TParserStateActionItem actionTPS_InXMLEntity[]
Definition: wparser_def.c:1110
ASCIIPARTHWORD
#define ASCIIPARTHWORD
Definition: wparser_def.c:46
actionTPS_InFURL
static const TParserStateActionItem actionTPS_InFURL[]
Definition: wparser_def.c:1431
p_iswhat
#define p_iswhat(type, nonascii)
Definition: wparser_def.c:388
actionTPS_InMantissaSign
static const TParserStateActionItem actionTPS_InMantissaSign[]
Definition: wparser_def.c:1089
mark_hl_fragments
static void mark_hl_fragments(HeadlineParsedText *prs, TSQuery query, List *locations, bool highlightall, int shortword, int min_words, int max_words, int max_fragments)
Definition: wparser_def.c:2226
WORD_T
#define WORD_T
Definition: wparser_def.c:37
TParserState
TParserState
Definition: wparser_def.c:120
TPS_InXMLEntityHexNumFirst
@ TPS_InXMLEntityHexNumFirst
Definition: wparser_def.c:144
TPS_InPort
@ TPS_InPort
Definition: wparser_def.c:167
TPS_InXMLEntityHexNum
@ TPS_InXMLEntityHexNum
Definition: wparser_def.c:145
TPS_InHostDomainSecond
@ TPS_InHostDomainSecond
Definition: wparser_def.c:164
TPS_InMantissaFirst
@ TPS_InMantissaFirst
Definition: wparser_def.c:137
TPS_InTagName
@ TPS_InTagName
Definition: wparser_def.c:150
TPS_InHyphenAsciiWordFirst
@ TPS_InHyphenAsciiWordFirst
Definition: wparser_def.c:185
TPS_Null
@ TPS_Null
Definition: wparser_def.c:198
TPS_InPathFirstFirst
@ TPS_InPathFirstFirst
Definition: wparser_def.c:174
TPS_InSignedIntFirst
@ TPS_InSignedIntFirst
Definition: wparser_def.c:126
TPS_InSignedInt
@ TPS_InSignedInt
Definition: wparser_def.c:127
TPS_InUnsignedInt
@ TPS_InUnsignedInt
Definition: wparser_def.c:125
TPS_InMantissa
@ TPS_InMantissa
Definition: wparser_def.c:139
TPS_InProtocolFirst
@ TPS_InProtocolFirst
Definition: wparser_def.c:182
TPS_InFURL
@ TPS_InFURL
Definition: wparser_def.c:181
TPS_InMantissaSign
@ TPS_InMantissaSign
Definition: wparser_def.c:138
TPS_InXMLBegin
@ TPS_InXMLBegin
Definition: wparser_def.c:148
TPS_InCommentEnd
@ TPS_InCommentEnd
Definition: wparser_def.c:162
TPS_InHyphenWordFirst
@ TPS_InHyphenWordFirst
Definition: wparser_def.c:187
TPS_InHyphenNumWordPart
@ TPS_InHyphenNumWordPart
Definition: wparser_def.c:196
TPS_InPortFirst
@ TPS_InPortFirst
Definition: wparser_def.c:166
TPS_InProtocolEnd
@ TPS_InProtocolEnd
Definition: wparser_def.c:184
TPS_InXMLEntityFirst
@ TPS_InXMLEntityFirst
Definition: wparser_def.c:140
TPS_InHyphenNumWordFirst
@ TPS_InHyphenNumWordFirst
Definition: wparser_def.c:189
TPS_InCommentLast
@ TPS_InCommentLast
Definition: wparser_def.c:158
TPS_InFileTwiddle
@ TPS_InFileTwiddle
Definition: wparser_def.c:172
TPS_InURLPathStart
@ TPS_InURLPathStart
Definition: wparser_def.c:179
TPS_InURLPathFirst
@ TPS_InURLPathFirst
Definition: wparser_def.c:178
TPS_InPathFirst
@ TPS_InPathFirst
Definition: wparser_def.c:173
TPS_InPathSecond
@ TPS_InPathSecond
Definition: wparser_def.c:175
TPS_InHyphenUnsignedInt
@ TPS_InHyphenUnsignedInt
Definition: wparser_def.c:197
TPS_InFileFirst
@ TPS_InFileFirst
Definition: wparser_def.c:171
TPS_InXMLEntityNumFirst
@ TPS_InXMLEntityNumFirst
Definition: wparser_def.c:142
TPS_InHyphenWordPart
@ TPS_InHyphenWordPart
Definition: wparser_def.c:194
TPS_InNumWord
@ TPS_InNumWord
Definition: wparser_def.c:122
TPS_InAsciiWord
@ TPS_InAsciiWord
Definition: wparser_def.c:123
TPS_InVersion
@ TPS_InVersion
Definition: wparser_def.c:136
TPS_InHost
@ TPS_InHost
Definition: wparser_def.c:169
TPS_InFile
@ TPS_InFile
Definition: wparser_def.c:176
TPS_InProtocolSecond
@ TPS_InProtocolSecond
Definition: wparser_def.c:183
TPS_InCloseCommentFirst
@ TPS_InCloseCommentFirst
Definition: wparser_def.c:160
TPS_InTagEscapeK
@ TPS_InTagEscapeK
Definition: wparser_def.c:153
TPS_InParseHyphenHyphen
@ TPS_InParseHyphenHyphen
Definition: wparser_def.c:193
TPS_InTagBackSleshed
@ TPS_InTagBackSleshed
Definition: wparser_def.c:155
TPS_InTagFirst
@ TPS_InTagFirst
Definition: wparser_def.c:147
TPS_InTagEnd
@ TPS_InTagEnd
Definition: wparser_def.c:156
TPS_InComment
@ TPS_InComment
Definition: wparser_def.c:159
TPS_InHyphenWord
@ TPS_InHyphenWord
Definition: wparser_def.c:188
TPS_InHyphenAsciiWord
@ TPS_InHyphenAsciiWord
Definition: wparser_def.c:186
TPS_InWord
@ TPS_InWord
Definition: wparser_def.c:124
TPS_InXMLEntityEnd
@ TPS_InXMLEntityEnd
Definition: wparser_def.c:146
TPS_InTagEscapeKK
@ TPS_InTagEscapeKK
Definition: wparser_def.c:154
TPS_InSpace
@ TPS_InSpace
Definition: wparser_def.c:128
TPS_InFileNext
@ TPS_InFileNext
Definition: wparser_def.c:177
TPS_InURLPath
@ TPS_InURLPath
Definition: wparser_def.c:180
TPS_Base
@ TPS_Base
Definition: wparser_def.c:121
TPS_InUDecimal
@ TPS_InUDecimal
Definition: wparser_def.c:130
TPS_InParseHyphen
@ TPS_InParseHyphen
Definition: wparser_def.c:192
TPS_InHostFirstAN
@ TPS_InHostFirstAN
Definition: wparser_def.c:168
TPS_InEmail
@ TPS_InEmail
Definition: wparser_def.c:170
TPS_InDecimalFirst
@ TPS_InDecimalFirst
Definition: wparser_def.c:131
TPS_InVersionFirst
@ TPS_InVersionFirst
Definition: wparser_def.c:135
TPS_InCloseCommentLast
@ TPS_InCloseCommentLast
Definition: wparser_def.c:161
TPS_InSVerVersion
@ TPS_InSVerVersion
Definition: wparser_def.c:134
TPS_InHyphenAsciiWordPart
@ TPS_InHyphenAsciiWordPart
Definition: wparser_def.c:195
TPS_InCommentFirst
@ TPS_InCommentFirst
Definition: wparser_def.c:157
TPS_InUDecimalFirst
@ TPS_InUDecimalFirst
Definition: wparser_def.c:129
TPS_InHostFirstDomain
@ TPS_InHostFirstDomain
Definition: wparser_def.c:163
TPS_InHostDomain
@ TPS_InHostDomain
Definition: wparser_def.c:165
TPS_InHyphenDigitLookahead
@ TPS_InHyphenDigitLookahead
Definition: wparser_def.c:191
TPS_InVerVersion
@ TPS_InVerVersion
Definition: wparser_def.c:133
TPS_InXMLEntityNum
@ TPS_InXMLEntityNum
Definition: wparser_def.c:143
TPS_InTag
@ TPS_InTag
Definition: wparser_def.c:152
TPS_InDecimal
@ TPS_InDecimal
Definition: wparser_def.c:132
TPS_InTagCloseFirst
@ TPS_InTagCloseFirst
Definition: wparser_def.c:149
TPS_InXMLEntity
@ TPS_InXMLEntity
Definition: wparser_def.c:141
TPS_InHyphenNumWord
@ TPS_InHyphenNumWord
Definition: wparser_def.c:190
TPS_InTagBeginEnd
@ TPS_InTagBeginEnd
Definition: wparser_def.c:151
mark_fragment
static void mark_fragment(HeadlineParsedText *prs, bool highlightall, int startpos, int endpos)
Definition: wparser_def.c:2139
actionTPS_InXMLEntityEnd
static const TParserStateActionItem actionTPS_InXMLEntityEnd[]
Definition: wparser_def.c:1149
actionTPS_InHyphenNumWord
static const TParserStateActionItem actionTPS_InHyphenNumWord[]
Definition: wparser_def.c:1494
actionTPS_InDecimal
static const TParserStateActionItem actionTPS_InDecimal[]
Definition: wparser_def.c:1046
A_POP
#define A_POP
Definition: wparser_def.c:222
actionTPS_InSignedIntFirst
static const TParserStateActionItem actionTPS_InSignedIntFirst[]
Definition: wparser_def.c:998
actionTPS_InTagEscapeK
static const TParserStateActionItem actionTPS_InTagEscapeK[]
Definition: wparser_def.c:1220
actionTPS_InSpace
static const TParserStateActionItem actionTPS_InSpace[]
Definition: wparser_def.c:1013
actionTPS_InFile
static const TParserStateActionItem actionTPS_InFile[]
Definition: wparser_def.c:1396
actionTPS_InHyphenAsciiWordPart
static const TParserStateActionItem actionTPS_InHyphenAsciiWordPart[]
Definition: wparser_def.c:1534
LASTNUM
#define LASTNUM
Definition: wparser_def.c:60
p_iseqC
static int p_iseqC(TParser *prs)
Definition: wparser_def.c:439
prsd_headline
Datum prsd_headline(PG_FUNCTION_ARGS)
Definition: wparser_def.c:2571
NUMHWORD
#define NUMHWORD
Definition: wparser_def.c:50
hlCover
static bool hlCover(HeadlineParsedText *prs, TSQuery query, List *locations, int *nextpos, int *p, int *q)
Definition: wparser_def.c:1987
SPACE
#define SPACE
Definition: wparser_def.c:47
actionTPS_InUDecimal
static const TParserStateActionItem actionTPS_InUDecimal[]
Definition: wparser_def.c:1031
TParserCharTest
int(* TParserCharTest)(struct TParser *)
Definition: wparser_def.c:204
actionTPS_InSignedInt
static const TParserStateActionItem actionTPS_InSignedInt[]
Definition: wparser_def.c:1004
p_isurlchar
static int p_isurlchar(TParser *prs)
Definition: wparser_def.c:463
actionTPS_InTagBeginEnd
static const TParserStateActionItem actionTPS_InTagBeginEnd[]
Definition: wparser_def.c:1192
actionTPS_InTagFirst
static const TParserStateActionItem actionTPS_InTagFirst[]
Definition: wparser_def.c:1153
TParserPosition
struct TParserPosition TParserPosition
NUMWORD
#define NUMWORD
Definition: wparser_def.c:38
FILEPATH
#define FILEPATH
Definition: wparser_def.c:54
actionTPS_InTagEscapeKK
static const TParserStateActionItem actionTPS_InTagEscapeKK[]
Definition: wparser_def.c:1227
p_isneC
static int p_isneC(TParser *prs)
Definition: wparser_def.c:445
EMAIL
#define EMAIL
Definition: wparser_def.c:39
actionTPS_InCommentLast
static const TParserStateActionItem actionTPS_InCommentLast[]
Definition: wparser_def.c:1252
newTParserPosition
static TParserPosition * newTParserPosition(TParserPosition *prev)
Definition: wparser_def.c:270
actionTPS_InHyphenWordPart
static const TParserStateActionItem actionTPS_InHyphenWordPart[]
Definition: wparser_def.c:1526
actionTPS_InMantissaFirst
static const TParserStateActionItem actionTPS_InMantissaFirst[]
Definition: wparser_def.c:1081
actionTPS_Base
static const TParserStateActionItem actionTPS_Base[]
Definition: wparser_def.c:926
SpecialHyphen
static void SpecialHyphen(TParser *prs)
Definition: wparser_def.c:554
mark_hl_words
static void mark_hl_words(HeadlineParsedText *prs, TSQuery query, List *locations, bool highlightall, int shortword, int min_words, int max_words)
Definition: wparser_def.c:2409
actionTPS_InHyphenNumWordFirst
static const TParserStateActionItem actionTPS_InHyphenNumWordFirst[]
Definition: wparser_def.c:1487
UNSIGNEDINT
#define UNSIGNEDINT
Definition: wparser_def.c:57
TParserSpecial
void(* TParserSpecial)(struct TParser *)
Definition: wparser_def.c:206
actionTPS_InEmail
static const TParserStateActionItem actionTPS_InEmail[]
Definition: wparser_def.c:1346
actionTPS_InXMLEntityHexNumFirst
static const TParserStateActionItem actionTPS_InXMLEntityHexNumFirst[]
Definition: wparser_def.c:1129
actionTPS_InURLPath
static const TParserStateActionItem actionTPS_InURLPath[]
Definition: wparser_def.c:1425
A_RERUN
#define A_RERUN
Definition: wparser_def.c:224
actionTPS_InSVerVersion
static const TParserStateActionItem actionTPS_InSVerVersion[]
Definition: wparser_def.c:1061
actionTPS_InAsciiWord
static const TParserStateActionItem actionTPS_InAsciiWord[]
Definition: wparser_def.c:954
tok_alias
static const char *const tok_alias[]
Definition: wparser_def.c:62
p_isstophost
static int p_isstophost(TParser *prs)
Definition: wparser_def.c:570
HLIDSKIP
#define HLIDSKIP(x)
Definition: wparser_def.c:1889
get_next_fragment
static void get_next_fragment(HeadlineParsedText *prs, int *startpos, int *endpos, int *curlen, int *poslen, int max_words)
Definition: wparser_def.c:2175
actionTPS_InHyphenUnsignedInt
static const TParserStateActionItem actionTPS_InHyphenUnsignedInt[]
Definition: wparser_def.c:1550
SIGNEDINT
#define SIGNEDINT
Definition: wparser_def.c:56
p_isasclet
static int p_isasclet(TParser *prs)
Definition: wparser_def.c:457
Actions
static const TParserStateAction Actions[]
Definition: wparser_def.c:1583
actionTPS_InXMLBegin
static const TParserStateActionItem actionTPS_InXMLBegin[]
Definition: wparser_def.c:1164
PARTHWORD
#define PARTHWORD
Definition: wparser_def.c:45
HLIDREPLACE
#define HLIDREPLACE(x)
Definition: wparser_def.c:1888
A_MERGE
#define A_MERGE
Definition: wparser_def.c:226
TParserInit
static TParser * TParserInit(char *str, int len)
Definition: wparser_def.c:287
actionTPS_InMantissa
static const TParserStateActionItem actionTPS_InMantissa[]
Definition: wparser_def.c:1095
actionTPS_InVersionFirst
static const TParserStateActionItem actionTPS_InVersionFirst[]
Definition: wparser_def.c:1068
p_isascii
static int p_isascii(TParser *prs)
Definition: wparser_def.c:451
actionTPS_InCommentFirst
static const TParserStateActionItem actionTPS_InCommentFirst[]
Definition: wparser_def.c:1243
actionTPS_InHyphenWord
static const TParserStateActionItem actionTPS_InHyphenWord[]
Definition: wparser_def.c:1478
p_isignore
static int p_isignore(TParser *prs)
Definition: wparser_def.c:581
actionTPS_InParseHyphenHyphen
static const TParserStateActionItem actionTPS_InParseHyphenHyphen[]
Definition: wparser_def.c:1519
actionTPS_InPort
static const TParserStateActionItem actionTPS_InPort[]
Definition: wparser_def.c:1320
TAG_T
#define TAG_T
Definition: wparser_def.c:48
actionTPS_InDecimalFirst
static const TParserStateActionItem actionTPS_InDecimalFirst[]
Definition: wparser_def.c:1040
URLPATH
#define URLPATH
Definition: wparser_def.c:53
prsd_lextype
Datum prsd_lextype(PG_FUNCTION_ARGS)
Definition: wparser_def.c:1833
ASCIIHWORD
#define ASCIIHWORD
Definition: wparser_def.c:51
HOST
#define HOST
Definition: wparser_def.c:41
actionTPS_InTag
static const TParserStateActionItem actionTPS_InTag[]
Definition: wparser_def.c:1198
prsd_start
Datum prsd_start(PG_FUNCTION_ARGS)
Definition: wparser_def.c:1851
A_BINGO
#define A_BINGO
Definition: wparser_def.c:221
TPARSERSTATEACTION
#define TPARSERSTATEACTION(state)
Definition: wparser_def.c:1575
TParserGet
static bool TParserGet(TParser *prs)
Definition: wparser_def.c:1665
XMLENTITY
#define XMLENTITY
Definition: wparser_def.c:58
p_ishost
static int p_ishost(TParser *prs)
Definition: wparser_def.c:587
prsd_end
Datum prsd_end(PG_FUNCTION_ARGS)
Definition: wparser_def.c:1873
A_CLRALL
#define A_CLRALL
Definition: wparser_def.c:227
p_isURLPath
static int p_isURLPath(TParser *prs)
Definition: wparser_def.c:615
SpecialVerVersion
static void SpecialVerVersion(TParser *prs)
Definition: wparser_def.c:561
actionTPS_InProtocolFirst
static const TParserStateActionItem actionTPS_InProtocolFirst[]
Definition: wparser_def.c:1437
actionTPS_InUnsignedInt
static const TParserStateActionItem actionTPS_InUnsignedInt[]
Definition: wparser_def.c:981
actionTPS_InUDecimalFirst
static const TParserStateActionItem actionTPS_InUDecimalFirst[]
Definition: wparser_def.c:1025
actionTPS_InTagCloseFirst
static const TParserStateActionItem actionTPS_InTagCloseFirst[]
Definition: wparser_def.c:1172
p_isEOF
static int p_isEOF(TParser *prs)
Definition: wparser_def.c:432
actionTPS_InCloseCommentLast
static const TParserStateActionItem actionTPS_InCloseCommentLast[]
Definition: wparser_def.c:1270
TParserCopyClose
static void TParserCopyClose(TParser *prs)
Definition: wparser_def.c:365
A_CLEAR
#define A_CLEAR
Definition: wparser_def.c:225
actionTPS_InFileFirst
static const TParserStateActionItem actionTPS_InFileFirst[]
Definition: wparser_def.c:1352
actionTPS_InNumWord
static const TParserStateActionItem actionTPS_InNumWord[]
Definition: wparser_def.c:943
actionTPS_InFileTwiddle
static const TParserStateActionItem actionTPS_InFileTwiddle[]
Definition: wparser_def.c:1362
TParserCopyInit
static TParser * TParserCopyInit(const TParser *orig)
Definition: wparser_def.c:319
actionTPS_InHost
static const TParserStateActionItem actionTPS_InHost[]
Definition: wparser_def.c:1335
A_PUSH
#define A_PUSH
Definition: wparser_def.c:223
actionTPS_InTagBackSleshed
static const TParserStateActionItem actionTPS_InTagBackSleshed[]
Definition: wparser_def.c:1234
actionTPS_InProtocolSecond
static const TParserStateActionItem actionTPS_InProtocolSecond[]
Definition: wparser_def.c:1443
actionTPS_InWord
static const TParserStateActionItem actionTPS_InWord[]
Definition: wparser_def.c:972
p_isspecial
static int p_isspecial(TParser *prs)
Definition: wparser_def.c:650
TParserClose
static void TParserClose(TParser *prs)
Definition: wparser_def.c:342
URL_T
#define URL_T
Definition: wparser_def.c:40
actionTPS_InXMLEntityNum
static const TParserStateActionItem actionTPS_InXMLEntityNum[]
Definition: wparser_def.c:1135
actionTPS_InVerVersion
static const TParserStateActionItem actionTPS_InVerVersion[]
Definition: wparser_def.c:1055
actionTPS_InHyphenAsciiWord
static const TParserStateActionItem actionTPS_InHyphenAsciiWord[]
Definition: wparser_def.c:1461
actionTPS_InXMLEntityHexNum
static const TParserStateActionItem actionTPS_InXMLEntityHexNum[]
Definition: wparser_def.c:1142
A_NEXT
#define A_NEXT
Definition: wparser_def.c:220
actionTPS_InPortFirst
static const TParserStateActionItem actionTPS_InPortFirst[]
Definition: wparser_def.c:1314
HWORD
#define HWORD
Definition: wparser_def.c:52
NUMPARTHWORD
#define NUMPARTHWORD
Definition: wparser_def.c:44
lex_descr
static const char *const lex_descr[]
Definition: wparser_def.c:89
INTERESTINGWORD
#define INTERESTINGWORD(j)
Definition: wparser_def.c:1901
SCIENTIFIC
#define SCIENTIFIC
Definition: wparser_def.c:42
SpecialTags
static void SpecialTags(TParser *prs)
Definition: wparser_def.c:522
actionTPS_InTagEnd
static const TParserStateActionItem actionTPS_InTagEnd[]
Definition: wparser_def.c:1239
actionTPS_InComment
static const TParserStateActionItem actionTPS_InComment[]
Definition: wparser_def.c:1258
actionTPS_InProtocolEnd
static const TParserStateActionItem actionTPS_InProtocolEnd[]
Definition: wparser_def.c:1449
actionTPS_InURLPathFirst
static const TParserStateActionItem actionTPS_InURLPathFirst[]
Definition: wparser_def.c:1415
actionTPS_InPathFirstFirst
static const TParserStateActionItem actionTPS_InPathFirstFirst[]
Definition: wparser_def.c:1381