diff --git a/lib/manager/bazel/extract.ts b/lib/manager/bazel/extract.ts index 74c3f3d1c407e010bcdcf8832c932f9284f07b1e..197e801b666bd8de882e4356263ac72f01a28dfa 100644 --- a/lib/manager/bazel/extract.ts +++ b/lib/manager/bazel/extract.ts @@ -85,23 +85,23 @@ const lexer = moo.states({ ].join('|') ), }, - unknown: { match: /[^]/, lineBreaks: true }, + unknown: moo.fallback, }, longDoubleQuoted: { stringFinish: { match: '"""', pop: 1 }, - char: { match: /[^]/, lineBreaks: true }, + char: moo.fallback, }, doubleQuoted: { stringFinish: { match: '"', pop: 1 }, - char: { match: /[^]/, lineBreaks: true }, + char: moo.fallback, }, longSingleQuoted: { stringFinish: { match: "'''", pop: 1 }, - char: { match: /[^]/, lineBreaks: true }, + char: moo.fallback, }, singleQuoted: { stringFinish: { match: "'", pop: 1 }, - char: { match: /[^]/, lineBreaks: true }, + char: moo.fallback, }, }); diff --git a/lib/manager/cake/index.ts b/lib/manager/cake/index.ts index 50d04007f97a92c7f889b0e645fa6bb0f3b96d34..f7352cb1f54eaebe273facbcc596fc471cc50b24 100644 --- a/lib/manager/cake/index.ts +++ b/lib/manager/cake/index.ts @@ -21,7 +21,7 @@ const lexer = moo.states({ match: /^#(?:addin|tool|module)\s+"(?:nuget|dotnet):[^"]+"\s*$/, value: (s: string) => s.trim().slice(1, -1), }, - unknown: { match: /[^]/, lineBreaks: true }, + unknown: moo.fallback, }, }); diff --git a/lib/manager/gradle-lite/common.ts b/lib/manager/gradle-lite/common.ts index 59ade8fc7f09206d58cd9d004ccb95c07ab00fe4..3ef9d3c020dccc2f40b7ce68c991634beafd0880 100644 --- a/lib/manager/gradle-lite/common.ts +++ b/lib/manager/gradle-lite/common.ts @@ -56,11 +56,10 @@ export enum TokenType { TripleDoubleQuotedStart = 'tripleDoubleQuotedStart', TripleQuotedFinish = 'tripleQuotedFinish', - Char = 'char', + Chars = 'chars', EscapedChar = 'escapedChar', String = 'string', - UnknownLexeme = 'unknownChar', UnknownFragment = 'unknownFragment', } diff --git a/lib/manager/gradle-lite/tokenizer.spec.ts b/lib/manager/gradle-lite/tokenizer.spec.ts index d7378abdd1aa08ee8cb98f0ae9ee5b857bbc7476..6c73c1f5ed30466566e5b05658ee012d4cdd8c70 100644 --- a/lib/manager/gradle-lite/tokenizer.spec.ts +++ b/lib/manager/gradle-lite/tokenizer.spec.ts @@ -35,7 +35,7 @@ describe(getName(), () => { TokenType.RightBrace, TokenType.RightBrace, ], - '@': [TokenType.UnknownLexeme], + '@': [TokenType.UnknownFragment], "'\\''": [ TokenType.SingleQuotedStart, TokenType.EscapedChar, @@ -54,23 +54,22 @@ describe(getName(), () => { ], "'x'": [ TokenType.SingleQuotedStart, - TokenType.Char, + TokenType.Chars, TokenType.SingleQuotedFinish, ], "'\n'": [ TokenType.SingleQuotedStart, - TokenType.Char, + TokenType.Chars, TokenType.SingleQuotedFinish, ], "'$x'": [ TokenType.SingleQuotedStart, - TokenType.Char, - TokenType.Char, + TokenType.Chars, TokenType.SingleQuotedFinish, ], "''''''": ['tripleQuotedStart', 'tripleQuotedFinish'], - "'''x'''": ['tripleQuotedStart', TokenType.Char, 'tripleQuotedFinish'], - "'''\n'''": ['tripleQuotedStart', TokenType.Char, 'tripleQuotedFinish'], + "'''x'''": ['tripleQuotedStart', TokenType.Chars, 'tripleQuotedFinish'], + "'''\n'''": ['tripleQuotedStart', TokenType.Chars, 'tripleQuotedFinish'], "'''\\''''": [ 'tripleQuotedStart', TokenType.EscapedChar, @@ -106,12 +105,12 @@ describe(getName(), () => { ], '"x"': [ TokenType.DoubleQuotedStart, - TokenType.Char, + TokenType.Chars, TokenType.DoubleQuotedFinish, ], '"\n"': [ TokenType.DoubleQuotedStart, - TokenType.Char, + TokenType.Chars, TokenType.DoubleQuotedFinish, ], // eslint-disable-next-line no-template-curly-in-string @@ -130,9 +129,7 @@ describe(getName(), () => { '"${x()}"': [ TokenType.DoubleQuotedStart, TokenType.IgnoredInterpolationStart, - TokenType.UnknownLexeme, - TokenType.UnknownLexeme, - TokenType.UnknownLexeme, + TokenType.UnknownFragment, TokenType.RightBrace, TokenType.DoubleQuotedFinish, ], @@ -140,7 +137,7 @@ describe(getName(), () => { '"${x{}}"': [ TokenType.DoubleQuotedStart, TokenType.IgnoredInterpolationStart, - TokenType.UnknownLexeme, + TokenType.UnknownFragment, TokenType.LeftBrace, TokenType.RightBrace, TokenType.RightBrace, diff --git a/lib/manager/gradle-lite/tokenizer.ts b/lib/manager/gradle-lite/tokenizer.ts index b37722fae2b87348b79b4b00dc94840027d8b4d8..271a80958336d825c7617ba9e5d5af4919240009 100644 --- a/lib/manager/gradle-lite/tokenizer.ts +++ b/lib/manager/gradle-lite/tokenizer.ts @@ -55,26 +55,26 @@ const lexer = moo.states({ match: '"', push: TokenType.DoubleQuotedStart, }, - [TokenType.UnknownLexeme]: { match: /./ }, + [TokenType.UnknownFragment]: moo.fallback, }, // Tokenize triple-quoted string literal characters [TokenType.TripleSingleQuotedStart]: { ...escapedChars, [TokenType.TripleQuotedFinish]: { match: "'''", pop: 1 }, - [TokenType.Char]: { match: /[^]/, lineBreaks: true }, + [TokenType.Chars]: moo.fallback, }, [TokenType.TripleDoubleQuotedStart]: { ...escapedChars, [TokenType.TripleQuotedFinish]: { match: '"""', pop: 1 }, - [TokenType.Char]: { match: /[^]/, lineBreaks: true }, + [TokenType.Chars]: moo.fallback, }, // Tokenize single-quoted string literal characters [TokenType.SingleQuotedStart]: { ...escapedChars, [TokenType.SingleQuotedFinish]: { match: "'", pop: 1 }, - [TokenType.Char]: { match: /[^]/, lineBreaks: true }, + [TokenType.Chars]: moo.fallback, }, // Tokenize double-quoted string literal chars and interpolations @@ -91,7 +91,7 @@ const lexer = moo.states({ match: /\${/, push: TokenType.IgnoredInterpolationStart, }, - [TokenType.Char]: { match: /[^]/, lineBreaks: true }, + [TokenType.Chars]: moo.fallback, }, // Ignore interpolation of complex expressionsË™, @@ -102,34 +102,17 @@ const lexer = moo.states({ push: TokenType.IgnoredInterpolationStart, }, [TokenType.RightBrace]: { match: '}', pop: 1 }, - [TokenType.UnknownLexeme]: { match: /[^]/, lineBreaks: true }, + [TokenType.UnknownFragment]: moo.fallback, }, }); -/* - Turn UnknownLexeme chars to UnknownFragment strings - */ -function processUnknownLexeme(acc: Token[], token: Token): Token[] { - if (token.type === TokenType.UnknownLexeme) { - const prevToken: Token = acc[acc.length - 1]; - if (prevToken?.type === TokenType.UnknownFragment) { - prevToken.value += token.value; - } else { - acc.push({ ...token, type: TokenType.UnknownFragment }); - } - } else { - acc.push(token); - } - return acc; -} - // -// Turn separated chars of string literal to single String token +// Turn substrings of chars and escaped chars into single String token // -function processChar(acc: Token[], token: Token): Token[] { +function processChars(acc: Token[], token: Token): Token[] { const tokenType = token.type; const prevToken: Token = acc[acc.length - 1]; - if ([TokenType.Char, TokenType.EscapedChar].includes(tokenType)) { + if ([TokenType.Chars, TokenType.EscapedChar].includes(tokenType)) { if (prevToken?.type === TokenType.String) { prevToken.value += token.value; } else { @@ -221,8 +204,7 @@ export function extractRawTokens(input: string): Token[] { export function processTokens(tokens: Token[]): Token[] { return tokens - .reduce(processUnknownLexeme, []) - .reduce(processChar, []) + .reduce(processChars, []) .reduce(processInterpolation, []) .filter(filterTokens); }