Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions packages/typescript/src/api/node/encoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -207,11 +207,11 @@ export function encodeNode(node: Node): Uint8Array {
const extendedDataValues: number[] = [];
const structuredWriter = new MsgpackWriter();

// We'll build an array of uint32 values for the nodes section, 7 per node
// We'll build an array of uint32 values for the nodes section, NODE_FIELDS per node
const nodeValues: number[] = [];

// Nil node (index 0)
nodeValues.push(0, 0, 0, 0, 0, 0, 0);
nodeValues.push(0, 0, 0, 0, 0, 0, 0, 0);

let nodeCount = 0;
let parentIndex = 0;
Expand All @@ -235,6 +235,7 @@ export function encodeNode(node: Node): Uint8Array {
parentIndex,
data,
node.flags,
0, // hasTrailingComma — not applicable to ordinary nodes
);

const saveParentIndex = parentIndex;
Expand Down Expand Up @@ -267,7 +268,8 @@ export function encodeNode(node: Node): Uint8Array {
0, // next
parentIndex,
list.length, // data for NodeList is its length
0, // flags
0, // flags — not applicable to NodeLists
list.hasTrailingComma ? 1 : 0,
);

const saveParentIndex = parentIndex;
Expand Down Expand Up @@ -313,6 +315,7 @@ export function encodeNode(node: Node): Uint8Array {
0,
rootData,
node.flags,
0, // hasTrailingComma — not applicable to ordinary nodes
);

const saveParent = parentIndex;
Expand Down
6 changes: 5 additions & 1 deletion packages/typescript/src/api/node/node.generated.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import {
NODE_OFFSET_DATA,
NODE_OFFSET_END,
NODE_OFFSET_FLAGS,
NODE_OFFSET_HAS_TRAILING_COMMA,
NODE_OFFSET_KIND,
NODE_OFFSET_NEXT,
NODE_OFFSET_PARENT,
Expand All @@ -43,7 +44,6 @@ export class RemoteNodeList extends Array<RemoteNode> implements NodeArray<Remot
}

parent: RemoteNode;
hasTrailingComma?: boolean;
transformFlags: number = 0;
protected view: DataView;
protected index: number;
Expand Down Expand Up @@ -71,6 +71,10 @@ export class RemoteNodeList extends Array<RemoteNode> implements NodeArray<Remot
return this.view.getUint32(this._byteIndex + NODE_OFFSET_DATA, true);
}

get hasTrailingComma(): boolean {
return this.view.getUint32(this._byteIndex + NODE_OFFSET_HAS_TRAILING_COMMA, true) !== 0;
}
Comment thread
mrazauskas marked this conversation as resolved.

private sourceFile: SourceFileInfo;

constructor(view: DataView, index: number, parent: RemoteNode, sourceFile: SourceFileInfo, offsetNodes: number) {
Expand Down
5 changes: 3 additions & 2 deletions packages/typescript/src/api/node/protocol.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
export const PROTOCOL_VERSION = 7;
export const PROTOCOL_VERSION = 8;
Comment thread
mrazauskas marked this conversation as resolved.

export const HEADER_OFFSET_METADATA = 0;
export const HEADER_OFFSET_HASH_LO0 = 4;
Expand All @@ -13,7 +13,7 @@ export const HEADER_OFFSET_STRUCTURED_DATA = 36;
export const HEADER_OFFSET_NODES = 40;
export const HEADER_SIZE = 44;

export const NODE_LEN = 28;
export const NODE_LEN = 32;
Comment thread
mrazauskas marked this conversation as resolved.

export const NODE_OFFSET_KIND = 0;
export const NODE_OFFSET_POS = 4;
Expand All @@ -22,6 +22,7 @@ export const NODE_OFFSET_NEXT = 12;
export const NODE_OFFSET_PARENT = 16;
export const NODE_OFFSET_DATA = 20;
export const NODE_OFFSET_FLAGS = 24;
export const NODE_OFFSET_HAS_TRAILING_COMMA = 28;

export const KIND_NODE_LIST = 0xFFFFFFFF;

Expand Down
24 changes: 24 additions & 0 deletions packages/typescript/test/async/api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
getSynthesizedDeepClone,
InternalSymbolName,
isCallExpression,
isExpressionStatement,
isFunctionDeclaration,
isIdentifier,
isImportDeclaration,
Expand Down Expand Up @@ -1083,6 +1084,29 @@ describe("SourceFile", () => {
});
});

describe("NodeArray", () => {
test("hasTrailingComma", async () => {
const api = spawnAPI({
"/tsconfig.json": JSON.stringify({ compilerOptions: { strict: true } }),
"/src/main.ts": `declare function foo(...args: any): void;\nfoo("a", "b",);\nfoo("a", "b");`,
});
try {
const snapshot = await api.updateSnapshot({ openProject: "/tsconfig.json" });
const project = snapshot.getProject("/tsconfig.json")!;
const sourceFile = await project.program.getSourceFile("/src/main.ts");
assert.ok(sourceFile);
const statements = sourceFile.statements.filter(isExpressionStatement);
assert.ok(isCallExpression(statements[0].expression));
assert.equal(statements[0].expression.arguments.hasTrailingComma, true);
assert.ok(isCallExpression(statements[1].expression));
assert.equal(statements[1].expression.arguments.hasTrailingComma, false);
}
finally {
await api.close();
}
});
});

test("unicode escapes", async () => {
const api = spawnAPI({
"/tsconfig.json": "{}",
Expand Down
6 changes: 3 additions & 3 deletions packages/typescript/test/encoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ describe("Encoder", () => {
// Verify header
const view = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength);
const metadata = view.getUint32(0, true);
assert.strictEqual(metadata >>> 24, 7, "protocol version should be 7");
assert.strictEqual(metadata >>> 24, 8, "protocol version should be 8");

// Verify we can decode it
const decoded = decode(encoded);
Expand Down Expand Up @@ -179,11 +179,11 @@ describe("Encoder", () => {
assert.strictEqual(rootKind, SyntaxKind.IfStatement);
});

test("protocol version is 7", () => {
test("protocol version is 8", () => {
const sf = makeSF("", "/test.ts", []);
const encoded = encodeSourceFile(sf);
const view = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength);
assert.strictEqual(view.getUint32(0, true) >>> 24, 7);
assert.strictEqual(view.getUint32(0, true) >>> 24, 8);
});

test("encodes source files without content mapping metadata", () => {
Expand Down
24 changes: 24 additions & 0 deletions packages/typescript/test/sync/api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
getSynthesizedDeepClone,
InternalSymbolName,
isCallExpression,
isExpressionStatement,
isFunctionDeclaration,
isIdentifier,
isImportDeclaration,
Expand Down Expand Up @@ -1091,6 +1092,29 @@ describe("SourceFile", () => {
});
});

describe("NodeArray", () => {
test("hasTrailingComma", () => {
const api = spawnAPI({
"/tsconfig.json": JSON.stringify({ compilerOptions: { strict: true } }),
"/src/main.ts": `declare function foo(...args: any): void;\nfoo("a", "b",);\nfoo("a", "b");`,
});
try {
const snapshot = api.updateSnapshot({ openProject: "/tsconfig.json" });
const project = snapshot.getProject("/tsconfig.json")!;
const sourceFile = project.program.getSourceFile("/src/main.ts");
assert.ok(sourceFile);
const statements = sourceFile.statements.filter(isExpressionStatement);
assert.ok(isCallExpression(statements[0].expression));
assert.equal(statements[0].expression.arguments.hasTrailingComma, true);
assert.ok(isCallExpression(statements[1].expression));
assert.equal(statements[1].expression.arguments.hasTrailingComma, false);
}
finally {
api.close();
}
});
});

test("unicode escapes", () => {
const api = spawnAPI({
"/tsconfig.json": "{}",
Expand Down
6 changes: 5 additions & 1 deletion tools/scripts/tsc/generate-encoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1491,6 +1491,7 @@ function emitNodeGeneratedImports(w: CodeWriter) {
w.write(` NODE_OFFSET_DATA,`);
w.write(` NODE_OFFSET_END,`);
w.write(` NODE_OFFSET_FLAGS,`);
w.write(` NODE_OFFSET_HAS_TRAILING_COMMA,`);
w.write(` NODE_OFFSET_KIND,`);
w.write(` NODE_OFFSET_NEXT,`);
w.write(` NODE_OFFSET_PARENT,`);
Expand Down Expand Up @@ -1518,7 +1519,6 @@ function emitRemoteNodeList(w: CodeWriter) {
w.write(` }`);
w.write(``);
w.write(` parent: RemoteNode;`);
w.write(` hasTrailingComma?: boolean;`);
w.write(` transformFlags: number = 0;`);
w.write(` protected view: DataView;`);
w.write(` protected index: number;`);
Expand Down Expand Up @@ -1546,6 +1546,10 @@ function emitRemoteNodeList(w: CodeWriter) {
w.write(` return this.view.getUint32(this._byteIndex + NODE_OFFSET_DATA, true);`);
w.write(` }`);
w.write(``);
w.write(` get hasTrailingComma(): boolean {`);
w.write(` return this.view.getUint32(this._byteIndex + NODE_OFFSET_HAS_TRAILING_COMMA, true) !== 0;`);
w.write(` }`);
w.write(``);
w.write(` private sourceFile: SourceFileInfo;`);
w.write(``);
w.write(` constructor(view: DataView, index: number, parent: RemoteNode, sourceFile: SourceFileInfo, offsetNodes: number) {`);
Expand Down
18 changes: 10 additions & 8 deletions tsc/internal/api/encoder/encoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ const (
NodeOffsetParent
NodeOffsetData
NodeOffsetFlags
NodeOffsetHasTrailingComma
// NodeSize is the number of bytes that represents a single node in the encoded format.
NodeSize
)
Expand Down Expand Up @@ -63,7 +64,7 @@ const (
)

const (
ProtocolVersion uint8 = 7
ProtocolVersion uint8 = 8
Comment thread
mrazauskas marked this conversation as resolved.
)

// Source File Binary Format
Expand All @@ -84,7 +85,7 @@ const (
// | String data | variable | UTF-8 encoded string data. |
// | Extended node data | variable | Extra data for some kinds of nodes. |
// | Structured data | variable | Msgpack-encoded metadata blobs (e.g. file references). |
// | Nodes | 28 bytes per node | Defines the AST structure of the file, with references to strings and extended data. |
// | Nodes | 32 bytes per node | Defines the AST structure of the file, with references to strings and extended data. |
//
// Header (44 bytes)
// -----------------
Expand Down Expand Up @@ -179,7 +180,7 @@ const (
//
// An offset of 0xFFFFFFFF indicates no data (empty array).
//
// Nodes (28 bytes per node)
// Nodes (32 bytes per node)
// -------------------------
//
// The nodes section contains the AST structure of the file. Nodes are represented in a flat array in source order,
Expand All @@ -195,8 +196,9 @@ const (
// | 16-20 | uint32 | Node index of parent |
// | 20-24 | | Node data |
// | 24-28 | uint32 | Node flags |
// | 28-32 | uint32 | HasTrailingComma (NodeList only; reserved/0 otherwise) |

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By the way, flags is always hard-coded to 0 for NodeList. So another solution could be to move HasTrailingComma to that slot for NodeList instead of creating a new one. (The ordinary nodes do not need HasTrailingComma.)

//
// The first 28 bytes of the nodes section are zeros representing a nil node, such that nodes without a parent or next
// The first 32 bytes of the nodes section are zeros representing a nil node, such that nodes without a parent or next
// sibling can unambiuously use `0` for those indices.
//
// NodeLists are represented as normal nodes with the special `kind` value `0xff_ff_ff_ff`. They are considered the parent
Expand Down Expand Up @@ -502,7 +504,7 @@ func encodeTree(rootNode *ast.Node, sourceFile *ast.SourceFile) ([]byte, *NodeIn
nodes[prevIndex*NodeSize+NodeOffsetNext+3] = b3
}

nodes = appendUint32s(nodes, SyntaxKindNodeList, utf16(nodeList.Pos()), utf16(nodeList.End()), 0, parentIndex, uint32(len(nodeList.Nodes)), 0)
nodes = appendUint32s(nodes, SyntaxKindNodeList, utf16(nodeList.Pos()), utf16(nodeList.End()), 0, parentIndex, uint32(len(nodeList.Nodes)), 0, uint32(boolToByte(nodeList.HasTrailingComma())))

saveParentIndex := parentIndex

Expand Down Expand Up @@ -535,7 +537,7 @@ func encodeTree(rootNode *ast.Node, sourceFile *ast.SourceFile) ([]byte, *NodeIn
nodes[prevIndex*NodeSize+NodeOffsetNext+3] = b3
}

nodes = appendUint32s(nodes, uint32(node.Kind), utf16(node.Pos()), utf16(node.End()), 0, parentIndex, getNodeData(node, strs, positionMap, &extendedData, &structuredData), uint32(node.Flags))
nodes = appendUint32s(nodes, uint32(node.Kind), utf16(node.Pos()), utf16(node.End()), 0, parentIndex, getNodeData(node, strs, positionMap, &extendedData, &structuredData), uint32(node.Flags), 0)

if nodeIndexMap != nil {
if _, ok := nodeIndexMap[node]; ok {
Expand All @@ -559,14 +561,14 @@ func encodeTree(rootNode *ast.Node, sourceFile *ast.SourceFile) ([]byte, *NodeIn
return node
}

nodes = appendUint32s(nodes, 0, 0, 0, 0, 0, 0, 0)
nodes = appendUint32s(nodes, 0, 0, 0, 0, 0, 0, 0, 0)

nodeCount++
parentIndex++
nodeTable = append(nodeTable, rootNode) // index 1 = root node

sfExtendedDataOffset = len(extendedData)
nodes = appendUint32s(nodes, uint32(rootNode.Kind), utf16(rootNode.Pos()), utf16(rootNode.End()), 0, 0, getNodeData(rootNode, strs, positionMap, &extendedData, &structuredData), uint32(rootNode.Flags))
nodes = appendUint32s(nodes, uint32(rootNode.Kind), utf16(rootNode.Pos()), utf16(rootNode.End()), 0, 0, getNodeData(rootNode, strs, positionMap, &extendedData, &structuredData), uint32(rootNode.Flags), 0)

visitor.VisitEachChild(rootNode)
if sourceFile != nil {
Expand Down
4 changes: 2 additions & 2 deletions tsc/internal/api/encoder/encoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ func TestEncodeSourceFile(t *testing.T) {

func TestEncodeContentMapperSourceFileMetadata(t *testing.T) {
t.Parallel()
if encoder.ProtocolVersion != 7 {
t.Fatalf("protocol version = %d, want 7", encoder.ProtocolVersion)
if encoder.ProtocolVersion != 8 {
t.Fatalf("protocol version = %d, want 8", encoder.ProtocolVersion)
}
sourceFile := parser.ParseSourceFile(ast.SourceFileParseOptions{
FileName: "/component.vue",
Expand Down