Merge branch 'develop' into main

Author: znorman-harris

CHANGELOG.md

@@ -34,6 +34,10 @@ Code style revamp! We reworked how routines, routine methods, properties, and st
 
 - When generating ENVI and IDL tasks, using our new case libraries, we attempt to make a pretty display name from parameter names. For example converting the keyword "my_keyword" to "My Keyword". This applied to task and parameter display names.
 
+## 4.1.2 December 2023
+
+Change the way we extract comments to reduce memory and speed up parsing by about 10%
+
 ## 4.1.1 December 2023
 
 Added a new preference for notebooks called "Quiet Mode" that allows you to control the IDL preference for `!quiet` when executing notebook cells.

libs/assembling/formatters/src/lib/fiddle/formatters/comment-formatter.ts

@@ -19,11 +19,6 @@ export const COMMENT_FORMATTER: TokenFormatter<CommentToken> = (
   if (meta.scope.slice(-1)[0] === TOKEN_NAMES.COMMENT_BLOCK) {
     token.match[0] = `;${token.match[0].substring(1)}`.trimEnd();
   } else {
-    // get everything by the first match, join together, remove excess comments
-    token.match[0] = `; ${token.match
-      .slice(1, token.match.length)
-      .map((el) => el.trim())
-      .join(' ')
-      .trim()}`;
+    token.match[0] = `; ${token.match[0].substring(1).trim()}`;
   }
 };

libs/assembling/styles/src/lib/default/style-routine-docs.ts

@@ -247,7 +247,7 @@ export function ReplaceRoutineDocs(parsed: IParsed, style: ICodeStyle) {
             type: BRANCH_TYPES.BASIC,
             name: TOKEN_NAMES.COMMENT,
             pos: [start + j, 0, docs[j].length],
-            match: [docs[j], docs[j].substring(2)],
+            match: [docs[j]],
             idx: j,
             parseProblems: [],
             scope: scope,

libs/parsing/syntax-validators/src/index.ts

@@ -3,7 +3,7 @@ export * from './lib/helpers/is-single-line';
 export * from './lib/validators/after-line-continuation';
 export * from './lib/validators/args-first';
 export * from './lib/validators/class-no-params';
-export * from './lib/validators/comments';
+export * from './lib/validators/comment-to-do';
 export * from './lib/validators/compile-opt';
 export * from './lib/validators/continue-break';
 export * from './lib/validators/dot-and-arrow';

libs/parsing/syntax-validators/src/lib/validators/comments.ts renamed to libs/parsing/syntax-validators/src/lib/validators/comment-to-do.ts

@@ -5,18 +5,28 @@ import {
 } from '@idl/parsing/syntax-tree';
 import { TOKEN_NAMES } from '@idl/parsing/tokenizer';
 
+/**
+ * Regex to extract TODO statements
+ */
+const TODO_REGEX = /^TODO:/im;
+
 /**
  * Extract TODO statements from comments
  */
 IDL_SYNTAX_TREE_VALIDATOR.onBasicToken(TOKEN_NAMES.COMMENT, (token, parsed) => {
+  /**
+   * Remove leading comment
+   */
+  const sub = token.match[0].substring(1).trim();
+
   // make sure we have a TODO which means three capture groups
   // see comments.spec.ts for some examples
-  if (token.match.length === 3) {
+  if (TODO_REGEX.test(sub)) {
     token.parseProblems.push(IDL_PROBLEM_CODES.TODO);
     parsed.parseProblems.push(
       SyntaxProblemWithoutTranslation(
         IDL_PROBLEM_CODES.TODO,
-        `TODO: ${token.match[2].trim()}`,
+        `TODO: ${sub.substring(5).trim()}`,
         token.pos,
         token.pos
       )

libs/parsing/tokenizer/src/lib/token-matches.interface.ts

@@ -24,7 +24,6 @@ import { AccessPropertyMatches } from './tokens/defs/access-property.interface';
 import {
   CommentBlockMatches,
   CommentMatches,
-  CommentWithToDoMatches,
 } from './tokens/defs/comment.interface';
 import {
   KeywordBinaryMatches,
@@ -64,7 +63,7 @@ export type TokenStartMatches<T extends TokenName> =
     : T extends CallProcedureMethodToken
     ? CallProcedureMethodMatches
     : T extends CommentToken
-    ? CommentMatches | CommentWithToDoMatches
+    ? CommentMatches
     : T extends CommentBlockToken
     ? CommentBlockMatches
     : T extends KeywordBinaryToken

libs/parsing/tokenizer/src/lib/tokens/defs/comment.interface.ts

@@ -5,37 +5,29 @@ import {
   TOKEN_NAMES,
 } from '../../tokens.interface';
 
+/**
+ * Token definition for comment
+ */
 export type CommentTokenDef = ITokenDef<CommentToken>;
 
 /**
  * For matching comments
  */
 export const COMMENT: CommentTokenDef = {
   name: TOKEN_NAMES.COMMENT,
-  match: /;\s*(TODO:)?(.*)$/im,
+  match: /;\s*.*$/im,
 };
 
 /**
- * Comment match without TODO
+ * Comment match
  *
  * @param {string} full Full match including the start of the comment
- * @param {string} comment Text after the semi-colon
  */
-export type CommentMatches = [full: string, comment: string];
+export type CommentMatches = [full: string];
 
 /**
- * Comment match with TODO
- *
- * @param {string} full Full match including the start of the comment
- * @param {string} todo TODO text, including semi-colon
- * @param {string} comment Statement to do
+ * Token definition for comment blocks
  */
-export type CommentWithToDoMatches = [
-  full: string,
-  todo: string,
-  comment: string
-];
-
 export type CommentBlockTokenDef = ITokenDef<CommentBlockToken>;
 
 /**

libs/shared/src/lib/version.interface.ts

@@ -1,4 +1,4 @@
 /**
  * Version of the extension
  */
-export const VERSION = '4.1.1';
+export const VERSION = '4.1.2';

libs/tests/assembler/src/lib/format.comments.1.spec.ts

@@ -54,7 +54,7 @@ describe(`[auto generated] Verify comment`, () => {
       const expectedFormatting: string[] = [
         `a = 42 ; comment OK`,
         `; ; comment bad, now fixed`,
-        `; TODO: something super crazy`,
+        `; TODO:   something super crazy`,
         `compile_opt idl2`,
         ``,
         `end`,

libs/tests/syntax-post-processors/src/lib/arg-defs.spec.ts

@@ -155,7 +155,7 @@ describe(`[auto generated] Correctly extract argument definitions from code`, ()
                     type: '1',
                     name: 15,
                     pos: [0, 39, 10],
-                    match: ['; commment', 'commment'],
+                    match: ['; commment'],
                     idx: 0,
                     scope: [72, 71, 31],
                     parseProblems: [],
@@ -421,7 +421,7 @@ describe(`[auto generated] Correctly extract argument definitions from code`, ()
                     type: '1',
                     name: 15,
                     pos: [0, 39, 10],
-                    match: ['; commment', 'commment'],
+                    match: ['; commment'],
                     idx: 0,
                     scope: [72, 71, 31],
                     parseProblems: [],