LSP: Implement semantic token support for CodeWindows #686

This also heavily refactors our existing scanner
infrastructure to accomodate both Antlr based lexers
and LSP-based semantic token providers.
As the latter also support supplying token modifiers,
support for this has been added as well
(e.g., rendering static things in italics).

Assets/SEE/Controls/Actions/ShowCodeAction.cs

@@ -9,6 +9,7 @@
 using UnityEngine;
 using SEE.DataModel.DG;
 using System;
+using Cysharp.Threading.Tasks;
 using SEE.UI.Window;
 using SEE.Utils.History;
 using SEE.Game.City;
@@ -254,36 +255,38 @@ public static CodeWindow ShowVCSDiff(GraphElementRef graphElementRef, DiffCity c
         public static CodeWindow ShowCode(GraphElementRef graphElementRef)
         {
             GraphElement graphElement = graphElementRef.Elem;
-            CodeWindow codeWindow;
-            if (graphElement.TryGetCommitID(out string commitID))
+            CodeWindow codeWindow = GetOrCreateCodeWindow(graphElementRef, graphElement.Filename);
+            EnterWindowContent().Forget();  // This can happen in the background.
+            return codeWindow;
+
+            async UniTaskVoid EnterWindowContent()
             {
-                codeWindow = GetOrCreateCodeWindow(graphElementRef, graphElement.Filename);
-                if (!graphElement.TryGetRepositoryPath(out string repositoryPath))
+                // We have to differentiate between a file-based and a VCS-based code city.
+                if (graphElement.TryGetCommitID(out string commitID))
                 {
-                    string message = $"Selected {GetName(graphElement)} has no repository path.";
-                    ShowNotification.Error("No repository path", message, log: false);
-                    throw new InvalidOperationException(message);
+                    if (!graphElement.TryGetRepositoryPath(out string repositoryPath))
+                    {
+                        string message = $"Selected {GetName(graphElement)} has no repository path.";
+                        ShowNotification.Error("No repository path", message, log: false);
+                        throw new InvalidOperationException(message);
+                    }
+                    IVersionControl vcs = VersionControlFactory.GetVersionControl(VCSKind.Git, repositoryPath);
+                    string[] fileContent = vcs.Show(graphElement.ID, commitID).
+                        Split("\\n", StringSplitOptions.RemoveEmptyEntries);
+                    codeWindow.EnterFromText(fileContent);
+                }
+                else
+                {
+                    await codeWindow.EnterFromFileAsync(GetPath(graphElement).absolutePlatformPath);
                 }
-                IVersionControl vcs = VersionControlFactory.GetVersionControl(VCSKind.Git, repositoryPath);
-                string[] fileContent = vcs.Show(graphElement.ID, commitID).
-                    Split("\\n", StringSplitOptions.RemoveEmptyEntries);
-                codeWindow.EnterFromText(fileContent);
-            }
-            else
-            {
-                (string filename, string absolutePlatformPath) = GetPath(graphElement);
-                codeWindow = GetOrCreateCodeWindow(graphElementRef, filename);
-                // File name of source code file to read from it
-                codeWindow.EnterFromFile(absolutePlatformPath);
-            }
 
-            // Pass line number to automatically scroll to it, if it exists
-            if (graphElement.SourceLine is { } line)
-            {
-                codeWindow.ScrolledVisibleLine = line;
+                // Pass line number to automatically scroll to it, if it exists
+                if (graphElement.SourceLine is { } line)
+                {
+                    codeWindow.ScrolledVisibleLine = line;
+                }
             }
 
-            return codeWindow;
         }
 
         public override bool Update()
@@ -299,6 +302,13 @@ public override bool Update()
                     return false;
                 }
 
+                ShowCodeWindow();
+            }
+
+            return false;
+
+            void ShowCodeWindow()
+            {
                 // Edges of type Clone will be handled differently. For these, we will be
                 // showing a unified diff.
                 CodeWindow codeWindow = graphElementRef is EdgeRef { Value: { Type: "Clone" } } edgeRef
@@ -313,8 +323,6 @@ public override bool Update()
                 manager.ActiveWindow = codeWindow;
                 // TODO (#669): Set font size etc in settings (maybe, or maybe that's too much)
             }
-
-            return false;
         }
     }
 }

Assets/SEE/DataModel/DG/IO/LSPImporter.cs

@@ -185,7 +185,7 @@ public async UniTask LoadAsync(Graph graph, Action<float> changePercentage = nul
                                        CancellationToken token = default)
         {
             // Query all documents whose file extension is supported by the language server.
-            List<string> relevantExtensions = Handler.Server.Languages.SelectMany(x => x.Extensions).ToList();
+            List<string> relevantExtensions = Handler.Server.Languages.SelectMany(x => x.FileExtensions).ToList();
             List<string> relevantDocuments = SourcePaths.SelectMany(RelevantDocumentsForPath)
                                                         .Where(x => ExcludedPaths.All(y => !x.StartsWith(y)))
                                                         .Distinct().ToList();

Assets/SEE/GraphProviders/LSPGraphProvider.cs

@@ -105,6 +105,13 @@ public class LSPGraphProvider : SingleGraphProvider
         [EnumToggleButtons, FoldoutGroup("Import Settings")]
         public DiagnosticKind IncludedDiagnosticLevels = DiagnosticKind.All;
 
+        /// <summary>
+        /// If true, LSP functionality will be available in code windows.
+        /// </summary>
+        [Tooltip("If true, LSP functionality will be available in code windows."), RuntimeTab(GraphProviderFoldoutGroup)]
+        [LabelWidth(150)]
+        public bool UseInCodeWindows = true;
+
         /// <summary>
         /// If true, the communication between the language server and SEE will be logged.
         /// </summary>
@@ -139,7 +146,7 @@ public class LSPGraphProvider : SingleGraphProvider
         /// <returns>The available language servers as a dropdown list.</returns>
         private IEnumerable<string> ServerDropdown()
         {
-            return LSPLanguage.All.Select(language => (language, LSPServer.All.Where(server => server.Languages.Contains(language))))
+            return LSPLanguage.AllLspLanguages.Select(language => (language, LSPServer.All.Where(server => server.Languages.Contains(language))))
                               .SelectMany(pair => pair.Item2.Select(server => $"{pair.language}/{server}"))
                               .OrderBy(server => server);
         }
@@ -248,6 +255,7 @@ public override async UniTask<Graph> ProvideAsync(Graph graph, AbstractSEECity c
             handler.Server = Server;
             handler.ProjectPath = ProjectPath.Path;
             handler.LogLSP = LogLSP;
+            handler.UseInCodeWindows = UseInCodeWindows;
             handler.TimeoutSpan = TimeSpan.FromSeconds(Timeout);
             await handler.InitializeAsync(executablePath: ServerPath ?? Server.ServerExecutable, token);
             if (token.IsCancellationRequested)

Assets/SEE/GraphProviders/VCSGraphProvider.cs

@@ -15,6 +15,7 @@
 using SEE.Scanner;
 using System.Threading;
 using Microsoft.Extensions.FileSystemGlobbing;
+using SEE.Scanner.Antlr;
 
 namespace SEE.GraphProviders
 {
@@ -264,7 +265,7 @@ public static Node BuildGraphFromPath(string path, Node parent, string parentPat
                 }
 
                 // Directory does not exist.
-                if (currentSegmentNode == null && pathSegments.Length > 1 && parent == null)
+                if (pathSegments.Length > 1 && parent == null)
                 {
                     rootNode.AddChild(NewNode(graph, pathSegments[0], directoryNodeType, pathSegments[0]));
                     return BuildGraphFromPath(nodePath, graph.GetNode(pathSegments[0]),
@@ -286,17 +287,15 @@ public static Node BuildGraphFromPath(string path, Node parent, string parentPat
                 }
 
                 // The node for the current pathSegment does not exist, and the node is a directory.
-                if (currentPathSegmentNode == null &&
-                    pathSegments.Length > 1)
+                if (pathSegments.Length > 1)
                 {
                     parent.AddChild(NewNode(graph, currentPathSegment, directoryNodeType, pathSegments[0]));
                     return BuildGraphFromPath(nodePath, graph.GetNode(currentPathSegment),
                         currentPathSegment, graph, rootNode);
                 }
 
                 // The node for the current pathSegment does not exist, and the node is a file.
-                if (currentPathSegmentNode == null &&
-                    pathSegments.Length == 1)
+                if (pathSegments.Length == 1)
                 {
                     Node addedFileNode = NewNode(graph, currentPathSegment, fileNodeType, pathSegments[0]);
                     parent.AddChild(addedFileNode);
@@ -342,21 +341,21 @@ private static List<string> ListTree(LibGit2Sharp.Tree tree)
         /// <param name="commitID">The commitID where the files exist.</param>
         /// <param name="language">The language the given text is written in.</param>
         /// <returns>The token stream for the specified file and commit.</returns>
-        public static IEnumerable<SEEToken> RetrieveTokens(string repositoryFilePath, Repository repository,
-                                                           string commitID, TokenLanguage language)
+        public static ICollection<AntlrToken> RetrieveTokens(string repositoryFilePath, Repository repository,
+                                                           string commitID, AntlrLanguage language)
         {
             Blob blob = repository.Lookup<Blob>($"{commitID}:{repositoryFilePath}");
 
             if (blob != null)
             {
                 string fileContent = blob.GetContentText();
-                return SEEToken.FromString(fileContent, language);
+                return AntlrToken.FromString(fileContent, language);
             }
             else
             {
                 // Blob does not exist.
                 Debug.LogWarning($"File {repositoryFilePath} does not exist.\n");
-                return Enumerable.Empty<SEEToken>();
+                return new List<AntlrToken>();
             }
         }
 
@@ -375,10 +374,10 @@ private static void AddCodeMetrics(Graph graph, Repository repository, string co
                 if (node.Type == fileNodeType)
                 {
                     string repositoryFilePath = node.ID;
-                    TokenLanguage language = TokenLanguage.FromFileExtension(Path.GetExtension(repositoryFilePath).TrimStart('.'));
-                    if (language != TokenLanguage.Plain)
+                    AntlrLanguage language = AntlrLanguage.FromFileExtension(Path.GetExtension(repositoryFilePath).TrimStart('.'));
+                    if (language != AntlrLanguage.Plain)
                     {
-                        IEnumerable<SEEToken> tokens = RetrieveTokens(repositoryFilePath, repository, commitID, language);
+                        ICollection<AntlrToken> tokens = RetrieveTokens(repositoryFilePath, repository, commitID, language);
                         node.SetInt(Metrics.Prefix + "LOC", TokenMetrics.CalculateLinesOfCode(tokens));
                         node.SetInt(Metrics.Prefix + "McCabe_Complexity", TokenMetrics.CalculateMcCabeComplexity(tokens));
                         TokenMetrics.HalsteadMetrics halsteadMetrics = TokenMetrics.CalculateHalsteadMetrics(tokens);

Assets/SEE/Net/Dashboard/Model/Issues/Issue.cs

@@ -39,7 +39,7 @@ public enum IssueState
         public IssueState State;
 
         /// <summary>
-        /// Whether or not the issue is suppressed or disabled via a control comment.
+        /// Whether the issue is suppressed or disabled via a control comment.
         /// </summary>
         /// <remarks>
         /// This column is only available for projects where importing of suppressed issues is configured

Assets/SEE/Scanner/Antlr.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 16346cc9b9ec430a9dfc96704b3868e2
+timeCreated: 1719432794