完成世界书、骰子、apiconfig页面处理

frontend/node_modules/langium/lib/parser/token-builder.js

@@ -0,0 +1,114 @@
+/******************************************************************************
+ * Copyright 2021 TypeFox GmbH
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ ******************************************************************************/
+import { Lexer } from 'chevrotain';
+import { isAbstractParserRule, isKeyword, isTerminalRule } from '../languages/generated/ast.js';
+import { streamAllContents } from '../utils/ast-utils.js';
+import { getAllReachableRules, terminalRegex } from '../utils/grammar-utils.js';
+import { escapeRegExp, isWhitespace, partialMatches } from '../utils/regexp-utils.js';
+import { stream } from '../utils/stream.js';
+export class DefaultTokenBuilder {
+    constructor() {
+        /**
+         * The list of diagnostics stored during the lexing process of a single text.
+         */
+        this.diagnostics = [];
+    }
+    buildTokens(grammar, options) {
+        const reachableRules = stream(getAllReachableRules(grammar, false));
+        const terminalTokens = this.buildTerminalTokens(reachableRules);
+        const tokens = this.buildKeywordTokens(reachableRules, terminalTokens, options);
+        // Add all terminals tokens to the end in the order they were defined
+        // Chevrotain documentation recommends to add Whitespace-like tokens at the start
+        // However, assuming the lexer is able to optimize the tokens, it should not matter
+        tokens.push(...terminalTokens);
+        // We don't need to add the EOF token explicitly.
+        // It is automatically available at the end of the token stream.
+        return tokens;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    flushLexingReport(text) {
+        return { diagnostics: this.popDiagnostics() };
+    }
+    popDiagnostics() {
+        const diagnostics = [...this.diagnostics];
+        this.diagnostics = [];
+        return diagnostics;
+    }
+    buildTerminalTokens(rules) {
+        return rules.filter(isTerminalRule).filter(e => !e.fragment)
+            .map(terminal => this.buildTerminalToken(terminal)).toArray();
+    }
+    buildTerminalToken(terminal) {
+        const regex = terminalRegex(terminal);
+        const pattern = this.requiresCustomPattern(regex) ? this.regexPatternFunction(regex) : regex;
+        const tokenType = {
+            name: terminal.name,
+            PATTERN: pattern,
+        };
+        if (typeof pattern === 'function') {
+            tokenType.LINE_BREAKS = true;
+        }
+        if (terminal.hidden) {
+            // Only skip tokens that are able to accept whitespace
+            tokenType.GROUP = isWhitespace(regex) ? Lexer.SKIPPED : 'hidden';
+        }
+        return tokenType;
+    }
+    requiresCustomPattern(regex) {
+        if (regex.flags.includes('u') || regex.flags.includes('s')) {
+            // Unicode and dotall regexes are not supported by Chevrotain.
+            return true;
+        }
+        else {
+            return false;
+        }
+    }
+    regexPatternFunction(regex) {
+        const stickyRegex = new RegExp(regex, regex.flags + 'y');
+        return (text, offset) => {
+            stickyRegex.lastIndex = offset;
+            const execResult = stickyRegex.exec(text);
+            return execResult;
+        };
+    }
+    buildKeywordTokens(rules, terminalTokens, options) {
+        return rules
+            // We filter by parser rules, since keywords in terminal rules get transformed into regex and are not actual tokens
+            .filter(isAbstractParserRule)
+            .flatMap(rule => streamAllContents(rule).filter(isKeyword))
+            .distinct(e => e.value).toArray()
+            // Sort keywords by descending length
+            .sort((a, b) => b.value.length - a.value.length)
+            .map(keyword => this.buildKeywordToken(keyword, terminalTokens, Boolean(options?.caseInsensitive)));
+    }
+    buildKeywordToken(keyword, terminalTokens, caseInsensitive) {
+        const keywordPattern = this.buildKeywordPattern(keyword, caseInsensitive);
+        const tokenType = {
+            name: keyword.value,
+            PATTERN: keywordPattern,
+            LONGER_ALT: this.findLongerAlt(keyword, terminalTokens)
+        };
+        if (typeof keywordPattern === 'function') {
+            tokenType.LINE_BREAKS = true;
+        }
+        return tokenType;
+    }
+    buildKeywordPattern(keyword, caseInsensitive) {
+        return caseInsensitive ?
+            new RegExp(escapeRegExp(keyword.value), 'i') :
+            keyword.value;
+    }
+    findLongerAlt(keyword, terminalTokens) {
+        return terminalTokens.reduce((longerAlts, token) => {
+            const pattern = token?.PATTERN;
+            if (pattern?.source && partialMatches('^' + pattern.source + '$', keyword.value)) {
+                longerAlts.push(token);
+            }
+            return longerAlts;
+        }, []);
+    }
+}
+//# sourceMappingURL=token-builder.js.map