From c6a9c223be9265f72b042dfcf41aad7006e9f9ba Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 30 Sep 2019 12:03:05 -0700 Subject: [PATCH 1/9] Remove stale reference --- .../Impl/Microsoft.Python.LanguageServer.csproj | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj index ca455737f..a33882e0d 100644 --- a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj +++ b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj @@ -33,14 +33,6 @@ - - - $(AnalysisReference)/Microsoft.Python.Analysis.Engine.dll - - - PreserveNewest - - From 6109ac761f1c2398c5ca6b4f38826cfb28051454 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 12 Nov 2019 13:55:07 -0800 Subject: [PATCH 2/9] Don't suppress LHS diagnostics on augmented assign --- .../UndefinedVariables/UndefinedVariablesWalker.cs | 2 -- src/Analysis/Ast/Test/AssignmentTests.cs | 9 +++++++++ src/Analysis/Ast/Test/LintUndefinedVarsTests.cs | 12 ++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs index 7365bf08d..42ac73bf0 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs @@ -49,9 +49,7 @@ public override bool Walk(SuiteStatement node) { HandleNonLocal(nls); break; case AugmentedAssignStatement augs: - _suppressDiagnostics = true; augs.Left?.Walk(new ExpressionWalker(this)); - _suppressDiagnostics = false; augs.Right?.Walk(new ExpressionWalker(this)); break; case AssignmentStatement asst: diff --git a/src/Analysis/Ast/Test/AssignmentTests.cs b/src/Analysis/Ast/Test/AssignmentTests.cs index cf3a54344..298454515 100644 --- a/src/Analysis/Ast/Test/AssignmentTests.cs +++ b/src/Analysis/Ast/Test/AssignmentTests.cs @@ -219,6 +219,15 @@ def __init__(self): .Which.Should().HaveMembers("abc", "y", "__class__"); } + [TestMethod, Priority(0)] + public async Task AugmentedAssignToUndefined() { + const string code = @" +x += 1 +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().NotHaveVariable("x"); + } + [TestMethod, Priority(0)] public async Task BaseInstanceVariable() { const string code = @" diff --git a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs index cdee6b70f..dec8c849a 100644 --- a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs +++ b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs @@ -821,6 +821,18 @@ class Subclass(MyClass): d.Should().BeEmpty(); } + [TestMethod, Priority(0)] + public async Task AugmentedAssignToUndefined() { + const string code = @" +x += 1 +"; + var d = await LintAsync(code); + d.Should().HaveCount(1); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(2, 1, 2, 2); + } + + private async Task> LintAsync(string code, InterpreterConfiguration configuration = null) { var analysis = await GetAnalysisAsync(code, configuration ?? PythonVersions.LatestAvailable3X); var a = Services.GetService(); From bcfc3b7246947674cbc18d7a76d2b90c105acf89 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 12 Nov 2019 15:30:27 -0800 Subject: [PATCH 3/9] Revert "Don't suppress LHS diagnostics on augmented assign" This reverts commit 6109ac761f1c2398c5ca6b4f38826cfb28051454. --- .../UndefinedVariables/UndefinedVariablesWalker.cs | 2 ++ src/Analysis/Ast/Test/AssignmentTests.cs | 9 --------- src/Analysis/Ast/Test/LintUndefinedVarsTests.cs | 12 ------------ 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs index 42ac73bf0..7365bf08d 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs @@ -49,7 +49,9 @@ public override bool Walk(SuiteStatement node) { HandleNonLocal(nls); break; case AugmentedAssignStatement augs: + _suppressDiagnostics = true; augs.Left?.Walk(new ExpressionWalker(this)); + _suppressDiagnostics = false; augs.Right?.Walk(new ExpressionWalker(this)); break; case AssignmentStatement asst: diff --git a/src/Analysis/Ast/Test/AssignmentTests.cs b/src/Analysis/Ast/Test/AssignmentTests.cs index 298454515..cf3a54344 100644 --- a/src/Analysis/Ast/Test/AssignmentTests.cs +++ b/src/Analysis/Ast/Test/AssignmentTests.cs @@ -219,15 +219,6 @@ def __init__(self): .Which.Should().HaveMembers("abc", "y", "__class__"); } - [TestMethod, Priority(0)] - public async Task AugmentedAssignToUndefined() { - const string code = @" -x += 1 -"; - var analysis = await GetAnalysisAsync(code); - analysis.Should().NotHaveVariable("x"); - } - [TestMethod, Priority(0)] public async Task BaseInstanceVariable() { const string code = @" diff --git a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs index dec8c849a..cdee6b70f 100644 --- a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs +++ b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs @@ -821,18 +821,6 @@ class Subclass(MyClass): d.Should().BeEmpty(); } - [TestMethod, Priority(0)] - public async Task AugmentedAssignToUndefined() { - const string code = @" -x += 1 -"; - var d = await LintAsync(code); - d.Should().HaveCount(1); - d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); - d[0].SourceSpan.Should().Be(2, 1, 2, 2); - } - - private async Task> LintAsync(string code, InterpreterConfiguration configuration = null) { var analysis = await GetAnalysisAsync(code, configuration ?? PythonVersions.LatestAvailable3X); var a = Services.GetService(); From c16646d97345765a93cac68179e467e9fd8ce697 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Thu, 12 Dec 2019 23:04:51 -0800 Subject: [PATCH 4/9] Escape [ and ] --- .../Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs | 5 ++--- .../Impl/Documentation/DocstringConverter.cs | 2 +- src/LanguageServer/Test/DocstringConverterTests.cs | 7 +++++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 5348dbc2d..d4b3b79d1 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -38,10 +38,9 @@ internal sealed class FunctionEvaluator : MemberEvaluator { public FunctionEvaluator(ExpressionEval eval, PythonFunctionOverload overload) : base(eval, overload.FunctionDefinition) { _overload = overload; - _function = overload.ClassMember ?? throw new NullReferenceException(nameof(overload.ClassMember)); + _function = overload.ClassMember ?? throw new ArgumentNullException(nameof(overload)); _self = _function.DeclaringType as PythonClassType; - - FunctionDefinition = overload.FunctionDefinition; + FunctionDefinition = overload.FunctionDefinition ?? throw new ArgumentNullException(nameof(overload)); } private FunctionDefinition FunctionDefinition { get; } diff --git a/src/LanguageServer/Impl/Documentation/DocstringConverter.cs b/src/LanguageServer/Impl/Documentation/DocstringConverter.cs index 046488540..3f345a11a 100644 --- a/src/LanguageServer/Impl/Documentation/DocstringConverter.cs +++ b/src/LanguageServer/Impl/Documentation/DocstringConverter.cs @@ -176,7 +176,7 @@ private static readonly (Regex, string)[] PotentialHeaders = new[] { private static readonly Regex TildaHeaderRegex = new Regex(@"^\s*~~~+$", RegexOptions.Singleline | RegexOptions.Compiled); private static readonly Regex PlusHeaderRegex = new Regex(@"^\s*\+\+\++$", RegexOptions.Singleline | RegexOptions.Compiled); private static readonly Regex LeadingAsteriskRegex = new Regex(@"^(\s+\* )(.*)$", RegexOptions.Singleline | RegexOptions.Compiled); - private static readonly Regex UnescapedMarkdownCharsRegex = new Regex(@"(? Date: Fri, 13 Dec 2019 11:09:45 -0800 Subject: [PATCH 5/9] PR feedback --- src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index d4b3b79d1..cba3040ca 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -38,9 +38,9 @@ internal sealed class FunctionEvaluator : MemberEvaluator { public FunctionEvaluator(ExpressionEval eval, PythonFunctionOverload overload) : base(eval, overload.FunctionDefinition) { _overload = overload; - _function = overload.ClassMember ?? throw new ArgumentNullException(nameof(overload)); + _function = overload.ClassMember ?? throw new ArgumentNullException(nameof(overload.ClassMember)); _self = _function.DeclaringType as PythonClassType; - FunctionDefinition = overload.FunctionDefinition ?? throw new ArgumentNullException(nameof(overload)); + FunctionDefinition = overload.FunctionDefinition ?? throw new ArgumentNullException(nameof(overload.FunctionDefinition)); } private FunctionDefinition FunctionDefinition { get; } From d5afe9738e7b2d2740e5d6d7c423f8f8178578a3 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 1 May 2020 00:50:29 -0700 Subject: [PATCH 6/9] Hightlight of unknown items (cherry picked from commit a1c140183b607f2e97e666336133674320502771) --- .../Impl/Sources/DocumentHighlightSource.cs | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs b/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs index afe1efd42..08f36dc3a 100644 --- a/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs +++ b/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs @@ -14,20 +14,17 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis; -using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Modules; -using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; -using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Documents; using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Sources { internal sealed class DocumentHighlightSource { @@ -40,7 +37,7 @@ public DocumentHighlightSource(IServiceContainer services) { public async Task DocumentHighlightAsync(Uri uri, SourceLocation location, CancellationToken cancellationToken = default) { if (uri == null) { - return Array.Empty(); + return null; } var analysis = await Document.GetAnalysisAsync(uri, _services, DocumentHighlightAnalysisTimeout, cancellationToken); @@ -48,17 +45,43 @@ public async Task DocumentHighlightAsync(Uri uri, SourceLoc var definition = definitionSource.FindDefinition(analysis, location, out var definingMember); if (definition == null || definingMember == null) { - return Array.Empty(); + return FromTokens(analysis, location); } var rootDefinition = definingMember.GetRootDefinition(); var result = rootDefinition.References .Where(r => r.DocumentUri.Equals(uri)) - .Select((r, i) => new DocumentHighlight { kind = (i == 0) ? DocumentHighlightKind.Write : DocumentHighlightKind.Read, range = r.Span }) + .Select((r, i) => new DocumentHighlight { + kind = i == 0 ? DocumentHighlightKind.Write : DocumentHighlightKind.Read, range = r.Span + }) .ToArray(); return result; } + + private static DocumentHighlight[] FromTokens(IDocumentAnalysis analysis, SourceLocation location) { + var position = analysis.Ast.LocationToIndex(location); + var content = analysis.Document.Content; + + var tokenizer = new Tokenizer(analysis.Document.Interpreter.LanguageVersion); + tokenizer.Initialize(null, new StringReader(content), SourceLocation.MinValue); + var tokens = tokenizer.ReadTokens(content.Length); + + var t = tokens.FirstOrDefault(x => x.SourceSpan.Start.Index <= position && position < x.SourceSpan.End.Index); + if (t.Category != TokenCategory.None) { + var length = t.SourceSpan.End.Index - t.SourceSpan.Start.Index; + return tokens + .Where(x => + x.SourceSpan.End.Index - x.SourceSpan.Start.Index == length && + string.Compare(content, x.SourceSpan.Start.Index, content, t.SourceSpan.Start.Index, length) == 0) + .Select(s => new DocumentHighlight { + kind = DocumentHighlightKind.Text, + range = s.SourceSpan + }).ToArray(); + } + + return null; + } } } From f1557115950c9e23886556309677320d19e07941 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 9 May 2020 13:19:33 -0700 Subject: [PATCH 7/9] Small fix --- src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs b/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs index 08f36dc3a..9ec2ee670 100644 --- a/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs +++ b/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs @@ -24,7 +24,6 @@ using Microsoft.Python.LanguageServer.Documents; using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Sources { internal sealed class DocumentHighlightSource { @@ -37,7 +36,7 @@ public DocumentHighlightSource(IServiceContainer services) { public async Task DocumentHighlightAsync(Uri uri, SourceLocation location, CancellationToken cancellationToken = default) { if (uri == null) { - return null; + return Array.Empty(); } var analysis = await Document.GetAnalysisAsync(uri, _services, DocumentHighlightAnalysisTimeout, cancellationToken); From e4da882cc815b4a3c344e59ecf2d978bc45a8ced Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 9 May 2020 18:00:22 -0700 Subject: [PATCH 8/9] Tests, token cache --- .../Impl/Sources/DocumentHighlightSource.cs | 67 +++++++++++++++++-- .../Test/DocumentHighlightTests.cs | 41 +++++++++++- 2 files changed, 99 insertions(+), 9 deletions(-) diff --git a/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs b/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs index 9ec2ee670..d7cca0af2 100644 --- a/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs +++ b/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs @@ -14,11 +14,15 @@ // permissions and limitations under the License. using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Documents; @@ -27,7 +31,7 @@ namespace Microsoft.Python.LanguageServer.Sources { internal sealed class DocumentHighlightSource { - private const int DocumentHighlightAnalysisTimeout = 10000; + private const int DocumentHighlightAnalysisTimeout = 1000; private readonly IServiceContainer _services; public DocumentHighlightSource(IServiceContainer services) { @@ -63,10 +67,7 @@ private static DocumentHighlight[] FromTokens(IDocumentAnalysis analysis, Source var position = analysis.Ast.LocationToIndex(location); var content = analysis.Document.Content; - var tokenizer = new Tokenizer(analysis.Document.Interpreter.LanguageVersion); - tokenizer.Initialize(null, new StringReader(content), SourceLocation.MinValue); - var tokens = tokenizer.ReadTokens(content.Length); - + var tokens = TokenCache.GetTokens(analysis.Document); var t = tokens.FirstOrDefault(x => x.SourceSpan.Start.Index <= position && position < x.SourceSpan.End.Index); if (t.Category != TokenCategory.None) { var length = t.SourceSpan.End.Index - t.SourceSpan.Start.Index; @@ -80,7 +81,61 @@ private static DocumentHighlight[] FromTokens(IDocumentAnalysis analysis, Source }).ToArray(); } - return null; + return Array.Empty(); + } + + private static class TokenCache { + private const int MaxEntries = 10; + private static readonly TimeSpan Expiration = TimeSpan.FromMinutes(30); + + private class Entry { + public DateTime AccessTime; + public WeakReference> Tokens; + } + + private static readonly Dictionary _cache = new Dictionary(); + + public static IReadOnlyList GetTokens(IDocument document) { + IReadOnlyList tokens; + long hash; + + var content = document.Content; + using (var sha = SHA1.Create()) { + hash = BitConverter.ToInt64(sha.ComputeHash(Encoding.UTF32.GetBytes(content))); + if (_cache.TryGetValue(hash, out var entry)) { + if (entry.Tokens.TryGetTarget(out tokens)) { + entry.AccessTime = DateTime.Now; + return tokens; + } + } + } + + var tokenizer = new Tokenizer(document.Interpreter.LanguageVersion); + using (var sr = new StringReader(content)) { + tokenizer.Initialize(null, sr, SourceLocation.MinValue); + tokens = tokenizer.ReadTokens(content.Length); + _cache[hash] = new Entry { + AccessTime = DateTime.Now, + Tokens = new WeakReference>(tokens) + }; + } + + var byTime = _cache.OrderByDescending(kvp => (DateTime.Now - kvp.Value.AccessTime).TotalSeconds).ToArray(); + + var expired = byTime.TakeWhile(kvp => DateTime.Now - kvp.Value.AccessTime > Expiration).ToArray(); + foreach (var e in expired) { + _cache.Remove(e.Key); + } + + if (_cache.Count > MaxEntries) { + var (key, _) = byTime.FirstOrDefault(); + if (key != default) { + _cache.Remove(key); + } + } + + return tokens; + } } } } diff --git a/src/LanguageServer/Test/DocumentHighlightTests.cs b/src/LanguageServer/Test/DocumentHighlightTests.cs index 4781b9db8..431da14d7 100644 --- a/src/LanguageServer/Test/DocumentHighlightTests.cs +++ b/src/LanguageServer/Test/DocumentHighlightTests.cs @@ -76,10 +76,45 @@ def func(x): [TestMethod, Priority(0)] public async Task HighlightEmptyDocument() { - await GetAnalysisAsync(string.Empty); + var analysis = await GetAnalysisAsync(string.Empty); var dhs = new DocumentHighlightSource(Services); - var references = await dhs.DocumentHighlightAsync(null, new SourceLocation(1, 1)); - references.Should().BeEmpty(); + var highlights = await dhs.DocumentHighlightAsync(analysis.Document.Uri, new SourceLocation(1, 1)); + highlights.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task HighlightNonReference() { + const string code = @" +x = y = 0 +assert x == 1 +assert y != 3 +"; + var analysis = await GetAnalysisAsync(code); + var dhs = new DocumentHighlightSource(Services); + var highlights = await dhs.DocumentHighlightAsync(analysis.Document.Uri, new SourceLocation(3, 5)); + + highlights.Should().HaveCount(2); + highlights[0].range.Should().Be(2, 0, 2, 6); + highlights[0].kind.Should().Be(DocumentHighlightKind.Text); + highlights[1].range.Should().Be(3, 0, 3, 6); + highlights[1].kind.Should().Be(DocumentHighlightKind.Text); + } + + [TestMethod, Priority(0)] + public async Task HighlightUndefined() { + const string code = @" +assert x == 1 +assert x != 3 +"; + var analysis = await GetAnalysisAsync(code); + var dhs = new DocumentHighlightSource(Services); + var highlights = await dhs.DocumentHighlightAsync(analysis.Document.Uri, new SourceLocation(2, 8)); + + highlights.Should().HaveCount(2); + highlights[0].range.Should().Be(1, 7, 1, 8); + highlights[0].kind.Should().Be(DocumentHighlightKind.Text); + highlights[1].range.Should().Be(2, 7, 2, 8); + highlights[1].kind.Should().Be(DocumentHighlightKind.Text); } } } From 415a98638b6104a7ce72b40b582d3dc8cabc58c6 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 9 May 2020 18:32:10 -0700 Subject: [PATCH 9/9] Cache tests --- .../Impl/Sources/DocumentHighlightSource.cs | 102 ++++++++++-------- .../Test/DocumentHighlightTests.cs | 42 +++++++- 2 files changed, 96 insertions(+), 48 deletions(-) diff --git a/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs b/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs index d7cca0af2..38a152614 100644 --- a/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs +++ b/src/LanguageServer/Impl/Sources/DocumentHighlightSource.cs @@ -22,7 +22,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis; -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Documents; @@ -32,6 +31,7 @@ namespace Microsoft.Python.LanguageServer.Sources { internal sealed class DocumentHighlightSource { private const int DocumentHighlightAnalysisTimeout = 1000; + private static TokenCache _tokenCache = new TokenCache(10, TimeSpan.FromMinutes(30)); private readonly IServiceContainer _services; public DocumentHighlightSource(IServiceContainer services) { @@ -67,7 +67,7 @@ private static DocumentHighlight[] FromTokens(IDocumentAnalysis analysis, Source var position = analysis.Ast.LocationToIndex(location); var content = analysis.Document.Content; - var tokens = TokenCache.GetTokens(analysis.Document); + var tokens = _tokenCache.GetTokens(analysis.Document.Content, analysis.Document.Interpreter.LanguageVersion); var t = tokens.FirstOrDefault(x => x.SourceSpan.Start.Index <= position && position < x.SourceSpan.End.Index); if (t.Category != TokenCategory.None) { var length = t.SourceSpan.End.Index - t.SourceSpan.Start.Index; @@ -83,58 +83,72 @@ private static DocumentHighlight[] FromTokens(IDocumentAnalysis analysis, Source return Array.Empty(); } + } - private static class TokenCache { - private const int MaxEntries = 10; - private static readonly TimeSpan Expiration = TimeSpan.FromMinutes(30); - private class Entry { - public DateTime AccessTime; - public WeakReference> Tokens; - } + internal class TokenCache { + internal class Entry { + public DateTime AccessTime; + public WeakReference> Tokens; + } - private static readonly Dictionary _cache = new Dictionary(); - - public static IReadOnlyList GetTokens(IDocument document) { - IReadOnlyList tokens; - long hash; - - var content = document.Content; - using (var sha = SHA1.Create()) { - hash = BitConverter.ToInt64(sha.ComputeHash(Encoding.UTF32.GetBytes(content))); - if (_cache.TryGetValue(hash, out var entry)) { - if (entry.Tokens.TryGetTarget(out tokens)) { - entry.AccessTime = DateTime.Now; - return tokens; - } - } - } + private readonly Dictionary _cache = new Dictionary(); + private readonly int _maxEntries; + private readonly TimeSpan _expiration; - var tokenizer = new Tokenizer(document.Interpreter.LanguageVersion); - using (var sr = new StringReader(content)) { - tokenizer.Initialize(null, sr, SourceLocation.MinValue); - tokens = tokenizer.ReadTokens(content.Length); - _cache[hash] = new Entry { - AccessTime = DateTime.Now, - Tokens = new WeakReference>(tokens) - }; + public TokenCache(int maxEntries, TimeSpan expiration) { + _maxEntries = maxEntries; + _expiration = expiration; + } + + public IReadOnlyList GetTokens(string content, PythonLanguageVersion languageVersion) { + IReadOnlyList tokens; + + var hash = GetHash(content); + if (_cache.TryGetValue(hash, out var entry)) { + if (entry.Tokens.TryGetTarget(out tokens)) { + entry.AccessTime = DateTime.Now; + return tokens; } + } - var byTime = _cache.OrderByDescending(kvp => (DateTime.Now - kvp.Value.AccessTime).TotalSeconds).ToArray(); + var tokenizer = new Tokenizer(languageVersion); + using (var sr = new StringReader(content)) { + tokenizer.Initialize(null, sr, SourceLocation.MinValue); + tokens = tokenizer.ReadTokens(content.Length); + _cache[hash] = new Entry { + AccessTime = DateTime.Now, + Tokens = new WeakReference>(tokens) + }; + } - var expired = byTime.TakeWhile(kvp => DateTime.Now - kvp.Value.AccessTime > Expiration).ToArray(); - foreach (var e in expired) { - _cache.Remove(e.Key); - } + var byTime = _cache.OrderByDescending(kvp => (DateTime.Now - kvp.Value.AccessTime).TotalSeconds).ToArray(); + + var expired = byTime.TakeWhile(kvp => DateTime.Now - kvp.Value.AccessTime > _expiration).ToArray(); + foreach (var e in expired) { + _cache.Remove(e.Key); + } - if (_cache.Count > MaxEntries) { - var (key, _) = byTime.FirstOrDefault(); - if (key != default) { - _cache.Remove(key); - } + if (_cache.Count > _maxEntries) { + var (key, _) = byTime.FirstOrDefault(); + if (key != default) { + _cache.Remove(key); } + } + + return tokens; + } + + // For tests + internal IEnumerable<(DateTime, IReadOnlyList)> Entries + => _cache.Values.Select(kvp => { + kvp.Tokens.TryGetTarget(out var t); + return (kvp.AccessTime, t); + }); - return tokens; + internal static long GetHash(string content) { + using (var sha = SHA1.Create()) { + return BitConverter.ToInt64(sha.ComputeHash(Encoding.UTF32.GetBytes(content))); } } } diff --git a/src/LanguageServer/Test/DocumentHighlightTests.cs b/src/LanguageServer/Test/DocumentHighlightTests.cs index 431da14d7..cfad14cfd 100644 --- a/src/LanguageServer/Test/DocumentHighlightTests.cs +++ b/src/LanguageServer/Test/DocumentHighlightTests.cs @@ -14,16 +14,15 @@ // permissions and limitations under the License. using System; -using System.IO; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using FluentAssertions; -using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.LanguageServer.Sources; using Microsoft.Python.LanguageServer.Tests.FluentAssertions; -using Microsoft.Python.Parsing.Tests; +using Microsoft.Python.Parsing; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -116,5 +115,40 @@ public async Task HighlightUndefined() { highlights[1].range.Should().Be(2, 7, 2, 8); highlights[1].kind.Should().Be(DocumentHighlightKind.Text); } + + [TestMethod, Priority(0)] + public void TokenCacheEntriesLimit() { + var c = new TokenCache(3, TimeSpan.FromMinutes(1)); + c.GetTokens("1", PythonLanguageVersion.V38); + c.GetTokens("12", PythonLanguageVersion.V38); + c.GetTokens("123", PythonLanguageVersion.V38); + c.GetTokens("1234", PythonLanguageVersion.V38); + + (DateTime AccessTime, IReadOnlyList Tokens)[] e = c.Entries.ToArray(); + + e.Should().HaveCount(3); + var byTime = e.OrderBy(x => x.AccessTime).ToArray(); + (byTime[0].Tokens[0].SourceSpan.End.Column - byTime[0].Tokens[0].SourceSpan.Start.Column).Should().Be(2); + (byTime[1].Tokens[0].SourceSpan.End.Column - byTime[1].Tokens[0].SourceSpan.Start.Column).Should().Be(3); + (byTime[2].Tokens[0].SourceSpan.End.Column - byTime[2].Tokens[0].SourceSpan.Start.Column).Should().Be(4); + } + + [TestMethod, Priority(0)] + public async Task TokenCacheExpiration() { + var c = new TokenCache(5, TimeSpan.FromMilliseconds(50)); + c.GetTokens("1", PythonLanguageVersion.V38); + await Task.Delay(10); + c.GetTokens("12", PythonLanguageVersion.V38); + await Task.Delay(10); + c.GetTokens("123", PythonLanguageVersion.V38); + await Task.Delay(40); + c.GetTokens("1234", PythonLanguageVersion.V38); + + (DateTime AccessTime, IReadOnlyList Tokens)[] e = c.Entries.ToArray(); + e.Should().HaveCount(2); + var byTime = e.OrderBy(x => x.AccessTime).ToArray(); + (byTime[0].Tokens[0].SourceSpan.End.Column - byTime[1].Tokens[0].SourceSpan.Start.Column).Should().Be(3); + (byTime[1].Tokens[0].SourceSpan.End.Column - byTime[1].Tokens[0].SourceSpan.Start.Column).Should().Be(4); + } } }