npm#

npm install tree-sitter-cli

二进制#

去GitHub的release界面下载你的平台的二进制文件

cargo#

cargo install tree-sitter-cli

文件夹#

之后的操作都在创建的文件夹进行

mkdir tree-sitter-${YOUR_LANGUAGE_NAME}
cd tree-sitter-${YOUR_LANGUAGE_NAME}

规则文件#

module.exports = grammar({
  name: 'YOUR_LANGUAGE_NAME',

  rules: {
    // TODO: add the actual grammar rules
    source_file: $ => 'hello'
  }
});

echo 'hello' > example.your-lang

tree-sitter parse example.your-lang

(source_file [0, 0] - [1, 0])

具体文档#

==================
Return statements
==================

func x() int {
  return 1;
}

---

(source_file
  (function_definition
    (identifier)
    (parameter_list)
    (primitive_type)
    (block
      (return_statement (number)))))

==================
Return statements
==================

func x() int {
  return 1;
}

---

(source_file
  (function_definition
    name: (identifier)
    parameters: (parameter_list)
    result: (primitive_type)
    body: (block
      (return_statement (number)))))

(source_file [0, 0] - [3, 0]
  (function_declaration [0, 0] - [2, 1]
    name: (identifier [0, 5] - [0, 9])
    parameters: (parameter_list [0, 9] - [0, 11])
    result: (type_identifier [0, 12] - [0, 15])
    body: (block [0, 16] - [2, 1]
      (return_statement [1, 2] - [1, 10]
        (expression_list [1, 9] - [1, 10]
          (int_literal [1, 9] - [1, 10]))))))

The Grammar DSL#

以下是一些内建方法 (built-in functions)，可以在grammar.js内定义规则。详细规则会在后面讲述。

The following is a complete list of built-in functions you can use in your grammar.js to define rules. Use-cases for some of these functions will be explained in more detail in later sections.

举例#

这里使用的Lua的规则来举例

module.exports = grammar({
    // 要解析的语言的名称
    name: 'lua',

    // 常用于空格和注释（此属性内的规则将被忽略）
    extras: $ => [
      $.comment,
      /[\s\n]/
    ],

    // 
    inline: $ => [
      $._statement
    ],

    conflicts: $ => [
      [$._prefix],
      [$._expression, $._variable_declarator],
      [$._expression, $.function_call_statement],
      [$.function_name, $.function_name_field]
    ],

    externals: $ => [
      $.comment,
      $.string
    ],

    rules: {
        nil: $ => 'nil',
        true: $ => 'true',
        false: $ => 'false',
    }
})

// `seq`代表按顺序匹配
table: $ => seq(
  // `{`和`}`是必须的
  '{',
  // `optional`代表规则`$._field_sequence`是可选的
  optional($._field_sequence),
  '}'
),
// `choice`从以下三个规则中选择一个
field: $ => choice(
  seq('[', $._expression, ']', '=', $._expression),
  seq($.identifier, '=', $._expression),
  $._expression
),
// `prec`会给规则设定优先级（这里是1，默认优先级为0）
_field_sequence: $ => prec(1, seq(
  $.field,
  repeat(seq($._field_sep, $.field)),
  optional($._field_sep)
)),
_field_sep: $ => choice(',', ';'),

{}
{a}
{1,2,3}
{"1",a,2}

for_statement: $ => seq(
  'for',
  // 可以通过alias(r1, r2)，将for_loop_expression或while_loop_expression
  // 规则重命名为loop_expression
  alias($._loop_expression, $.loop_expression),
  'do',
  repeat($._statement),
  optional($.return_statement),
  'end'
),

-- for $.loop_expression do
for i=1,10 do
  -- 重复0次或以上$._statement
  print(i)\
-- end
end

初始化#

选择你要解析的语言的解析器，你可以clone已经实现了的解析器或者使用上一节自己创建的解析器。

git clone https://github.com/tree-sitter/tree-sitter-go
git clone https://github.com/tree-sitter/tree-sitter-javascript
git clone https://github.com/tree-sitter/tree-sitter-python

使用 Language.build_library 方法将它们编译成可从 Python 使用的库。如果自上次修改源代码后库已经编译，则此函数将立即返回：

from tree_sitter import Language, Parser

Language.build_library(
  # Store the library in the `build` directory
  'build/my-languages.so',

  # Include one or more languages
  [
    'vendor/tree-sitter-go',
    'vendor/tree-sitter-javascript',
    'vendor/tree-sitter-python'
  ]
)

解析#

创建一个解析器，并设定语言

parser = Parser()
parser.set_language(PY_LANGUAGE)

进行解析

tree = parser.parse(bytes("""
def foo():
    if bar:
        baz()
""", "utf8"))
# 或者
src = bytes("""
def foo():
    if bar:
        baz()
""", "utf8")

def read_callable(byte_offset, point):
    return src[byte_offset:byte_offset+1]

tree = parser.parse(read_callable)

检查语法树

root_node = tree.root_node
assert root_node.type == 'module'
assert root_node.start_point == (1, 0)
assert root_node.end_point == (3, 13)

function_node = root_node.children[0]
assert function_node.type == 'function_definition'
assert function_node.child_by_field_name('name').type == 'identifier'

function_name_node = function_node.children[1]
assert function_name_node.type == 'identifier'
assert function_name_node.start_point == (1, 4)
assert function_name_node.end_point == (1, 7)

assert root_node.sexp() == "(module "
    "(function_definition "
        "name: (identifier) "
        "parameters: (parameters) "
        "body: (block "
            "(if_statement "
                "condition: (identifier) "
                "consequence: (block "
                    "(expression_statement (call "
                        "function: (identifier) "
                        "arguments: (argument_list))))))))"

使用遍历（walk）方式

cursor = tree.walk()

assert cursor.node.type == 'module'

assert cursor.goto_first_child()
assert cursor.node.type == 'function_definition'

assert cursor.goto_first_child()
assert cursor.node.type == 'def'

# Returns `False` because the `def` node has no children
assert not cursor.goto_first_child()

assert cursor.goto_next_sibling()
assert cursor.node.type == 'identifier'

assert cursor.goto_next_sibling()
assert cursor.node.type == 'parameters'

assert cursor.goto_parent()
assert cursor.node.type == 'function_definition'

创建查询#

from tree_sitter import Language, Parser, Query

# 初始化解析器和语言
parser = Parser()
parser.set_language(PY_LANGUAGE)

# 解析源代码
source_code = bytes("""
def factorial(n):
    if n <= 1:
        return 1
    else:
        return n * factorial(n - 1)
""", "utf8")
tree = parser.parse(source_code)

# 创建查询来找到所有函数定义
query_string = """
(function_definition
  name: (identifier) @function_name
  parameters: (parameters) @params
  body: (block) @body)
"""
query = Query(PY_LANGUAGE, query_string)

# 执行查询
captures = query.captures(tree.root_node)

# 处理结果
for capture in captures:
    node, tag = capture
    if tag == "function_name":
        name = source_code[node.start_byte:node.end_byte].decode('utf8')
        print(f"发现函数: {name}")

查询模式#

1. 节点匹配：(node_type)
2. 字段约束：field_name: (node_type)
3. 捕获：@capture_name
4. 量词：*, +, ? (零或多次、一次或多次、可选)
5. 分组：使用圆括号分组模式
6. 否定：! 匹配不符合指定模式的节点
7. 谓词：施加额外的约束条件

例如，查找所有函数调用：

(call
  function: (identifier) @function_name
  arguments: (argument_list))

代码高亮#

通过分析语法树，你可以准确地识别代码的各个部分（关键字、标识符、字符串等），并应用相应的颜色样式。

def highlight_python_code(source_code):
    parser = Parser()
    parser.set_language(PY_LANGUAGE)
    tree = parser.parse(bytes(source_code, "utf8"))
    
    # 定义高亮查询
    highlight_query = Query(PY_LANGUAGE, """
        (function_definition name: (identifier) @function)
        (call function: (identifier) @function.call)
        "def" @keyword
        "if" @keyword
        "else" @keyword
        "return" @keyword
        (string) @string
        (integer) @number
        (comment) @comment
    """)
    
    # 应用高亮
    captures = highlight_query.captures(tree.root_node)
    highlights = []
    for node, tag in captures:
        highlights.append({
            'type': tag,
            'start': node.start_byte,
            'end': node.end_byte,
            'text': source_code[node.start_byte:node.end_byte]
        })
    
    return highlights

静态分析#

利用语法树，你可以编写强大的静态代码分析工具，用于检查代码质量、发现潜在问题或执行自定义规则。

def find_todo_comments(source_code):
    parser = Parser()
    parser.set_language(PY_LANGUAGE)
    tree = parser.parse(bytes(source_code, "utf8"))
    
    todo_query = Query(PY_LANGUAGE, """
        (comment) @comment
    """)
    
    todos = []
    for node, _ in todo_query.captures(tree.root_node):
        comment_text = source_code[node.start_byte:node.end_byte].decode('utf8')
        if "TODO" in comment_text or "FIXME" in comment_text:
            line, col = node.start_point
            todos.append({
                'text': comment_text,
                'line': line + 1,  # 1-indexed line numbers
                'column': col
            })
    
    return todos

代码重构#

通过结合语法树分析和文本编辑，你可以实现复杂的代码重构功能。

def rename_function(source_code, old_name, new_name):
    parser = Parser()
    parser.set_language(PY_LANGUAGE)
    tree = parser.parse(bytes(source_code, "utf8"))
    
    # 查找所有函数名称和调用
    query = Query(PY_LANGUAGE, """
        (function_definition name: (identifier) @function_def)
        (call function: (identifier) @function_call)
    """)
    
    # 收集需要重命名的节点
    edits = []
    for node, tag in query.captures(tree.root_node):
        name = source_code[node.start_byte:node.end_byte].decode('utf8')
        if name == old_name:
            edits.append((node.start_byte, node.end_byte))
    
    # 从后向前应用编辑（避免偏移问题）
    edits.sort(reverse=True)
    result = source_code.decode('utf8')
    for start, end in edits:
        result = result[:start] + new_name + result[end:]
    
    return result

编辑器集成#

Tree-sitter 已经集成到许多流行的代码编辑器中，如 Visual Studio Code（通过插件）、Neovim、Emacs 等。