apoloz/codespaces
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ed58bd043b711a22338c5ca12e7c6f469a019fd1
codespaces/scripts/build_belief_map.py
Anatoliy Poloz ed58bd043b feat(php): php projects support added
2026-06-22 08:12:55 +03:00

4434 lines
161 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
"""
Belief Map Builder -- Architecture Graph Generator.
Generates ``belief_map.sexp``, a flat S-expression graph of all modules,
entities, and relationships in a codebase. Each line is a self-contained
fact, queryable with ``rg`` or ``scripts/belief_search.py``.
Based on the Theory of Code Space (ToCS) methodology for reducing the
"Active-Passive Gap" in AI code agents.
Parsers
-------
- **TypeScript/TSX**: tree-sitter (``tree-sitter-typescript``). Handles
decorators, heritage, all methods, object literals, arrow functions.
- **Python**: stdlib ``ast``. Full class/function/import extraction.
- **PHP**: tree-sitter (``tree-sitter-php``). Handles classes, interfaces,
traits, PHP 8.1 enums, attributes (``#[...]``), heritage, trait usage,
and Symfony/Laravel constructor DI. Falls back to regex parser if
``tree-sitter-php`` is unavailable.
Modes
-----
**Default** (fast, ~5-8s for full workspace):
Parses all .py/.ts/.tsx/.php files, builds import/reference/data-flow edges.
**LSP-enhanced** (``--lsp``, ~1-5min):
After default pass, starts ``typescript-language-server``,
``pyright-langserver``, and ``phpactor`` per project to query call
hierarchy and references. Adds precise ``calls`` edges and ``:lsp``
annotations.
Edge Types
----------
- ``imports`` -- module imports module. ``:via-base`` if target exports
abstract types.
- ``calls-api`` -- implements/extends abstract type. ``:via-ifc``.
- ``data-flow`` -- data flows to validation module. ``:validated``.
- ``refs`` -- entity-level cross-file reference (``Module::Entity``).
- ``calls`` (LSP only) -- function calls function with call-site lines.
Output Format
-------------
Flat S-expression facts (``belief_map.sexp``)::
(node module/id ts "purpose description")
(cls module/id ClassName 49 (:deco Injectable) (:methods create update delete))
(fn module/id functionName 126)
(imports module/a module/b :via-base)
(refs module/a module/b::EntityName)
(calls module/a::fnA module/b::fnB :lines 52 67 :lsp)
Requirements
------------
::
pip3 install tree-sitter tree-sitter-typescript tree-sitter-python tree-sitter-php
Optional for ``--lsp``: ``typescript-language-server``, ``pyright-langserver``,
``phpactor`` (``composer global require phpactor/phpactor``).
Usage
-----
::
python3 scripts/build_belief_map.py # incremental (~5-8s)
python3 scripts/build_belief_map.py --full # full rebuild
python3 scripts/build_belief_map.py --lsp # with call hierarchy
python3 scripts/build_belief_map.py ./app-service # target sub-repo
"""
from __future__ import annotations
import ast
import hashlib
import json
import os
import re
import subprocess
import sys
import threading
import time
from concurrent.futures import ProcessPoolExecutor, as_completed
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Optional
from urllib.parse import quote, unquote
import tree_sitter_typescript as _tst
from tree_sitter import Language as _TsLanguage, Parser as _TsParser
_TS_LANG = _TsLanguage(_tst.language_typescript())
_TSX_LANG = _TsLanguage(_tst.language_tsx())
# Optional tree-sitter-php grammar. If unavailable, parse_php() falls back to
# a regex-based parser (see parse_php_regex). PHP files remain visible in the
# graph, but with reduced entity/edge precision.
try:
import tree_sitter_php as _tsp
_PHP_LANG = _TsLanguage(_tsp.language_php())
except (ImportError, AttributeError):
_PHP_LANG = None
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
SKIP_DIRS = {
"node_modules",
".venv",
"venv",
"__pycache__",
".git",
"dist",
".next",
".turbo",
"build",
"coverage",
".tox",
"site-packages",
".mypy_cache",
".pytest_cache",
".ruff_cache",
"target",
".worktrees",
".ralphy-worktrees",
".ralphy-sandboxes",
# PHP: vendor = composer dependencies, var = Symfony cache/logs.
# (Laravel storage/ and bootstrap/ are NOT skipped -- they may contain
# real source like bootstrap/app.php.)
"vendor",
"var",
}
CACHE_FILE = ".belief_map_cache.json"
OUTPUT_FILE = ".belief_map.sexp"
# Default timeout for LSP requests (seconds)
LSP_REQUEST_TIMEOUT = 15.0
# Timeout for LSP server initialization (seconds)
LSP_INIT_TIMEOUT = 60.0
# -- Python regex patterns --
PY_ABC_RE = re.compile(r"""class\s+\w+\s*\(.*(?:ABC|ABCMeta|Protocol).*\)""")
PY_INHERITS_RE = re.compile(r"""class\s+(\w+)\s*\(([^)]+)\)""")
# Naming convention detectors
SNAKE_CASE_RE = re.compile(r"^[a-z][a-z0-9_]*$")
CAMEL_CASE_RE = re.compile(r"^[a-z][a-zA-Z0-9]*$")
PASCAL_CASE_RE = re.compile(r"^[A-Z][a-zA-Z0-9]*$")
KEBAB_CASE_RE = re.compile(r"^[a-z][a-z0-9-]*$")
# ---------------------------------------------------------------------------
# Data types
# ---------------------------------------------------------------------------
@dataclass
class Entity:
name: str
kind: str # class, function, interface, type, enum, trait, method, decorator
line: int
methods: list[str] # method names (for classes)
decorators: list[str] # decorator names
bases: list[str] # parent classes / implemented interfaces / used traits
def to_dict(self) -> dict:
d: dict = {"name": self.name, "kind": self.kind, "line": self.line}
if self.methods:
d["methods"] = self.methods
if self.decorators:
d["decorators"] = self.decorators
if self.bases:
d["bases"] = self.bases
return d
@dataclass
class ImportedName:
"""A single name imported from another module."""
local_name: str # name used locally
original_name: str # name in source module ("default" for default imports)
module: str # raw import specifier
def to_dict(self) -> dict:
return {
"local": self.local_name,
"original": self.original_name,
"module": self.module,
}
@dataclass
class FileResult:
path: str
language: str
repo: str
mtime: float
content_hash: str
imports: list[str]
exports_abstract: list[str]
implements: list[str]
extends: list[str]
purpose: str
naming_convention: str
has_validation: bool
entities: list[dict] # Entity.to_dict() results
imported_names: list[dict] # ImportedName.to_dict() results
exported_names: list[str] # names exported from this file
traits_used: list[str] = field(
default_factory=list
) # PHP: trait names used inside class bodies
@dataclass
class LspEdge:
"""An edge discovered by LSP analysis."""
source: str # "node_id::entity_name" or "node_id"
target: str # "node_id::entity_name" or "node_id"
edge_type: str # "CALLS", "REFERENCES"
metadata: dict = field(default_factory=dict)
@dataclass
class ProjectGroup:
"""A group of files belonging to the same LSP project."""
root: str # absolute path to project root
language: str # "typescript" or "python"
config_file: str # path to tsconfig.json or pyrightconfig.json
files: list[FileResult] = field(default_factory=list)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def file_hash(path: str) -> str:
try:
st = os.stat(path)
with open(path, "rb") as f:
chunk = f.read(8192)
return hashlib.md5(chunk + str(st.st_size).encode()).hexdigest()
except OSError:
return ""
def detect_naming(filename: str) -> str:
stem = Path(filename).stem
for suffix in (".test", ".spec", ".e2e", ".stories", ".d"):
if stem.endswith(suffix):
stem = stem[: -len(suffix)]
if SNAKE_CASE_RE.match(stem):
return "snake_case"
if KEBAB_CASE_RE.match(stem):
return "kebab-case"
if PASCAL_CASE_RE.match(stem):
return "PascalCase"
if CAMEL_CASE_RE.match(stem):
return "camelCase"
return "mixed"
def _levenshtein(a: str, b: str) -> int:
"""Compute Levenshtein edit distance between two strings."""
if len(a) < len(b):
return _levenshtein(b, a)
if not b:
return len(a)
prev = list(range(len(b) + 1))
for i, ca in enumerate(a):
curr = [i + 1]
for j, cb in enumerate(b):
curr.append(min(prev[j + 1] + 1, curr[j] + 1, prev[j] + (ca != cb)))
prev = curr
return prev[-1]
def _fuzzy_suffix_match(name: str, suffix: str, max_dist: int = 1) -> bool:
"""Check if the tail of `name` fuzzy-matches `suffix` within edit distance."""
if len(suffix) < 6:
return False # too short for fuzzy -- "Store"/"Storey" false positives
tail = name[-len(suffix) - max_dist :] # grab suffix + slack for insertions
return _levenshtein(tail.lower(), suffix.lower()) <= max_dist
def infer_purpose(
path: str, content: str, language: str, entities: Optional[list] = None
) -> str:
"""Infer architectural purpose from path, content, and entity names."""
parts = []
rel = path.lower()
# Path-based heuristics
if "/domain/" in rel:
parts.append("domain logic")
elif "/infrastructure/" in rel or "/infra/" in rel:
parts.append("infrastructure adapter")
elif "/api/" in rel:
parts.append("API layer")
elif "/app/" in rel or "/application/" in rel:
parts.append("application service")
elif "/components/" in rel or "/ui/" in rel:
parts.append("UI component")
elif "/hooks/" in rel:
parts.append("React hook")
elif "/pages/" in rel or "/routes/" in rel:
parts.append("page/route")
elif "/migrations/" in rel:
parts.append("database migration")
elif "/test" in rel or ".test." in rel or ".spec." in rel:
parts.append("test")
elif "/config" in rel:
parts.append("configuration")
elif "/utils/" in rel or "/helpers/" in rel or "/commons/" in rel:
parts.append("shared utilities")
elif "/repositories/" in rel or "/repo/" in rel:
parts.append("repository")
elif "/services/" in rel:
parts.append("service")
elif "/dto/" in rel:
parts.append("DTO")
elif "/guards/" in rel or "guard" in rel:
parts.append("access guard")
elif "/store/" in rel or "/stores/" in rel:
parts.append("state store")
elif "/schema" in rel:
parts.append("schema")
elif "/sdk/" in rel:
parts.append("SDK")
elif "/api-client/" in rel or "/api_client/" in rel:
parts.append("API client")
# PHP-specific path conventions (Laravel / Symfony)
if language == "php":
if (
"/Http/Controllers/" in rel
or "/Controller/" in rel
or "/Controllers/" in rel
):
parts.append("API controller")
elif "/Http/Middleware/" in rel or "/Middleware/" in rel:
parts.append("middleware")
elif "/Models/" in rel or "/Entity/" in rel or "/Entities/" in rel:
parts.append("domain logic")
elif "/Services/" in rel:
parts.append("service")
elif "/Repositories/" in rel:
parts.append("repository")
elif "/Resources/" in rel:
parts.append("API resource")
elif "/Requests/" in rel:
parts.append("request validation")
elif "/Providers/" in rel:
parts.append("service provider")
elif "/Console/Commands/" in rel or "/Command/" in rel:
parts.append("command")
elif "/Events/" in rel:
parts.append("event")
elif "/Listeners/" in rel:
parts.append("event listener")
elif "/Jobs/" in rel:
parts.append("worker job")
elif "/Migrations/" in rel or "/database/migrations" in rel:
parts.append("database migration")
elif "/routes/" in rel or "/config/" in rel:
parts.append("configuration")
# Docstring extraction
head = content[:2048]
if language == "python":
match = re.search(r'^"""(.*?)"""', head, re.DOTALL)
if not match:
match = re.search(r"^'''(.*?)'''", head, re.DOTALL)
if match:
parts.append(match.group(1).strip().split("\n")[0][:120])
else:
match = re.search(r"/\*\*\s*\n?\s*\*?\s*(.+?)(?:\n|\*/)", head)
if match:
parts.append(match.group(1).strip()[:120])
# Entity-name heuristics (Gap 3 fix: reduces "general module" fallback)
# -- Suffix -> (purpose label, dedup keyword) --
_ENTITY_SUFFIX_MAP = [
# Core architecture roles
("Service", "service", "service"),
("Controller", "API controller", "controller"),
("Repository", "repository", "repository"),
("Repo", "repository", "repository"),
("Gateway", "gateway", "gateway"),
("Registry", "registry", "registry"),
("Store", "state store", "store"),
("Guard", "access guard", "guard"),
("Module", "module", "module"),
("Adapter", "adapter", "adapter"),
("Provider", "provider", "provider"),
("Processor", "processor", "processor"),
("Agent", "agent", "agent"),
("Resolver", "resolver", "resolver"),
("Scheduler", "scheduler", "scheduler"),
("Worker", "worker", "worker"),
("Client", "client", "client"),
("Connector", "connector", "connector"),
("Manager", "manager", "manager"),
("Subscriber", "subscriber", "subscriber"),
("Dispatcher", "dispatcher", "dispatcher"),
("Emitter", "emitter", "emitter"),
("Interceptor", "interceptor", "interceptor"),
("Pipe", "validation pipe", "pipe"),
("Handler", "handler", "handler"),
("Listener", "event listener", "listener"),
# GoF patterns
("Factory", "Factory pattern", "factory"),
("Builder", "Builder pattern", "builder"),
("Singleton", "Singleton pattern", "singleton"),
("Observer", "Observer pattern", "observer"),
("Strategy", "Strategy pattern", "strategy"),
("Command", "Command pattern", "command"),
("Mediator", "Mediator pattern", "mediator"),
("Visitor", "Visitor pattern", "visitor"),
("Decorator", "Decorator pattern", "decorator"),
("Proxy", "Proxy pattern", "proxy"),
("Facade", "Facade pattern", "facade"),
("Composite", "Composite pattern", "composite"),
("Iterator", "Iterator pattern", "iterator"),
("ChainHandler", "Chain of Responsibility", "chain"),
# Fowler / DDD patterns
("Specification", "Specification pattern", "specification"),
("Spec", "Specification pattern", "specification"),
("ValueObject", "value object", "value"),
("Aggregate", "aggregate root", "aggregate"),
("UnitOfWork", "Unit of Work", "unitofwork"),
("DomainEvent", "domain event", "domainevent"),
("IdentityMap", "Identity Map", "identitymap"),
("Policy", "policy", "policy"),
("Plugin", "plugin", "plugin"),
# SDK / API
("Sdk", "SDK", "sdk"),
("SDK", "SDK", "sdk"),
("Api", "API client", "api client"),
("API", "API client", "api client"),
]
if entities:
joined_lower = " ".join(parts).lower()
for ent in entities:
ename = ent.get("name", "") if isinstance(ent, dict) else ent.name
ekind = ent.get("kind", "") if isinstance(ent, dict) else ent.kind
if ekind not in ("class", "interface"):
continue
# Exact suffix match first
matched = False
for suffix, purpose_label, dedup_kw in _ENTITY_SUFFIX_MAP:
if ename.endswith(suffix):
if dedup_kw not in joined_lower:
parts.append(purpose_label)
joined_lower = " ".join(parts).lower()
matched = True
break
# Fuzzy suffix fallback (distance <= 1, suffixes >= 4 chars)
if not matched and len(ename) >= 5:
for suffix, purpose_label, dedup_kw in _ENTITY_SUFFIX_MAP:
if _fuzzy_suffix_match(ename, suffix):
if dedup_kw not in joined_lower:
parts.append(purpose_label)
joined_lower = " ".join(parts).lower()
break
# Pattern detection -- only flag patterns when the file DEFINES one,
# not when it merely imports from a module named factory/strategy.
joined_check = " ".join(parts).lower()
# GoF Creational
if "factory" not in joined_check:
if re.search(
r"class \w*Factory|def (?:create|make|build)_\w+.*->|def get_\w+.*factory",
head,
):
parts.append("Factory pattern")
if "strategy" not in joined_check:
if re.search(r"class \w*Strategy|strategy_map|STRATEGIES", head):
parts.append("Strategy pattern")
if "builder" not in joined_check:
if re.search(r"class \w+Builder\b", head):
parts.append("Builder pattern")
if "singleton" not in joined_check:
if re.search(
r"getInstance\s*\(|_instance\s*[:=]|private\s+static\s+instance", head
):
parts.append("Singleton pattern")
# GoF Structural
if "composite" not in joined_check:
if re.search(r"class \w+Composite\b|addChild\s*\(|getChildren\s*\(", head):
parts.append("Composite pattern")
if "proxy" not in joined_check:
if re.search(r"class \w+Proxy\b", head):
parts.append("Proxy pattern")
if "facade" not in joined_check:
if re.search(r"class \w+Facade\b", head):
parts.append("Facade pattern")
# GoF Behavioral
if "observer" not in joined_check:
if re.search(r"class \w+Observer\b|\.subscribe\s*\(|EventEmitter", head):
parts.append("Observer pattern")
if "command" not in joined_check:
if re.search(r"class \w+Command\b", head):
parts.append("Command pattern")
if "mediator" not in joined_check:
if re.search(r"class \w+Mediator\b", head):
parts.append("Mediator pattern")
if "visitor" not in joined_check:
if re.search(r"class \w+Visitor\b|\.accept\s*\(\s*visitor", head):
parts.append("Visitor pattern")
if "chain" not in joined_check:
if re.search(r"class \w+Chain\w*\b|setNext\s*\(|handleNext\s*\(", head):
parts.append("Chain of Responsibility")
if "memento" not in joined_check:
if re.search(r"class \w+Memento\b", head):
parts.append("Memento pattern")
# Fowler / DDD patterns
if "specification" not in joined_check:
if re.search(r"class \w+Specification\b|isSatisfiedBy\s*\(", head):
parts.append("Specification pattern")
if "aggregate" not in joined_check:
if re.search(r"class \w+Aggregate\b", head):
parts.append("aggregate root")
if "unitofwork" not in joined_check.replace(" ", ""):
if re.search(r"class \w+UnitOfWork\b", head):
parts.append("Unit of Work")
if "domainevent" not in joined_check.replace(" ", ""):
if re.search(
r"class \w+DomainEvent\b|class \w+Event\b(?!Emitter|Handler|Listener)", head
):
parts.append("domain event")
if "identitymap" not in joined_check.replace(" ", ""):
if re.search(r"class \w+IdentityMap\b", head):
parts.append("Identity Map")
if "value object" not in joined_check:
if re.search(r"class \w+ValueObject\b", head):
parts.append("value object")
# NestJS decorators -- only label when entity-name heuristics didn't
# provide a more specific purpose (controller, service, gateway, etc.)
_specific_kws = {
"controller",
"service",
"repository",
"guard",
"gateway",
"adapter",
"provider",
"processor",
"registry",
"agent",
"resolver",
"scheduler",
"worker",
"store",
"handler",
"interceptor",
"pipe",
"listener",
"subscriber",
"dispatcher",
"emitter",
"factory",
"builder",
"observer",
"strategy",
"command",
"mediator",
"visitor",
"proxy",
"facade",
"composite",
"specification",
"aggregate",
"policy",
"plugin",
"manager",
"client",
"connector",
"sdk",
"api client",
}
joined_after = " ".join(parts).lower()
has_specific = any(kw in joined_after for kw in _specific_kws)
if not has_specific:
if re.search(r"@Controller\b", head):
parts.append("API controller")
elif re.search(r"@Module\b", head):
parts.append("NestJS wiring module")
elif re.search(r"@Injectable\b", head):
parts.append("NestJS injectable")
if re.search(r"Router|router\s*\(|createRouter", head):
if "router" not in " ".join(parts).lower():
parts.append("router")
if re.search(r"middleware|Middleware", head):
if "middleware" not in " ".join(parts).lower():
parts.append("middleware")
# --- File-stem based heuristics ---
stem = Path(path).stem.lower()
for sfx in (".test", ".spec", ".e2e", ".stories", ".d"):
if stem.endswith(sfx):
stem = stem[: -len(sfx)]
if not parts:
if stem.endswith("dto") or stem.endswith("dtos"):
parts.append("data transfer object")
elif stem.endswith("mapper") or stem.endswith("mappers"):
parts.append("data mapper")
elif (
stem.endswith("helpers")
or stem.endswith("helper")
or stem.endswith("utils")
or stem.endswith("util")
):
parts.append("utility helpers")
elif stem.endswith("constants") or stem.endswith("const"):
parts.append("constants")
elif stem.endswith("types") or stem.endswith("type"):
parts.append("type definitions")
elif stem.endswith("interfaces") or stem.endswith("interface"):
parts.append("interface definitions")
elif stem.endswith("enums") or stem.endswith("enum"):
parts.append("enum definitions")
elif stem.endswith("models") or stem.endswith("model"):
parts.append("data model")
elif stem.endswith("entities") or stem.endswith("entity"):
parts.append("domain entity")
elif (
stem.endswith("errors")
or stem.endswith("error")
or stem.endswith("exceptions")
or stem.endswith("exception")
):
parts.append("error definitions")
elif stem.endswith("factories") or stem.endswith("factory"):
parts.append("factory")
elif stem.endswith("validators") or stem.endswith("validator"):
parts.append("validation")
elif stem.endswith("handlers") or stem.endswith("handler"):
parts.append("event handler")
elif stem.endswith("listeners") or stem.endswith("listener"):
parts.append("event listener")
elif stem.endswith("middleware"):
parts.append("middleware")
elif stem.endswith("interceptor"):
parts.append("interceptor")
elif stem.endswith("guard"):
parts.append("access guard")
elif stem.endswith("pipe"):
parts.append("validation pipe")
elif stem.endswith("decorators") or stem.endswith("decorator"):
parts.append("decorator")
elif stem.endswith("migration"):
parts.append("database migration")
elif stem.endswith("seed") or stem.endswith("seeder"):
parts.append("database seed")
elif stem.endswith("fixtures") or stem.endswith("fixture"):
parts.append("test fixture")
elif stem.endswith("mocks") or stem.endswith("mock"):
parts.append("test mock")
# Core architecture roles (stem-based)
elif stem.endswith("controller") or stem.endswith("controllers"):
parts.append("API controller")
elif stem.endswith("service") or stem.endswith("services"):
parts.append("service")
elif stem.endswith("repository") or stem.endswith("repositories"):
parts.append("repository")
elif stem.endswith("gateway") or stem.endswith("gateways"):
parts.append("gateway")
elif stem.endswith("registry") or stem.endswith("registries"):
parts.append("registry")
elif stem.endswith("agent") or stem.endswith("agents"):
parts.append("agent")
elif stem.endswith("store") or stem.endswith("stores"):
parts.append("state store")
elif stem.endswith("policy") or stem.endswith("policies"):
parts.append("policy")
elif stem.endswith("resolver") or stem.endswith("resolvers"):
parts.append("resolver")
elif stem.endswith("scheduler") or stem.endswith("schedulers"):
parts.append("scheduler")
elif stem.endswith("worker") or stem.endswith("workers"):
parts.append("worker")
elif stem.endswith("client") or stem.endswith("clients"):
parts.append("client")
elif stem.endswith("connector") or stem.endswith("connectors"):
parts.append("connector")
elif stem.endswith("manager") or stem.endswith("managers"):
parts.append("manager")
elif stem.endswith("subscriber") or stem.endswith("subscribers"):
parts.append("subscriber")
elif stem.endswith("dispatcher") or stem.endswith("dispatchers"):
parts.append("dispatcher")
elif stem.endswith("emitter") or stem.endswith("emitters"):
parts.append("emitter")
elif stem.endswith("adapter") or stem.endswith("adapters"):
parts.append("adapter")
elif stem.endswith("provider") or stem.endswith("providers"):
parts.append("provider")
elif stem.endswith("processor") or stem.endswith("processors"):
parts.append("processor")
# GoF pattern stems
elif stem.endswith("observer"):
parts.append("Observer pattern")
elif stem.endswith("builder"):
parts.append("Builder pattern")
elif stem.endswith("visitor"):
parts.append("Visitor pattern")
elif stem.endswith("mediator"):
parts.append("Mediator pattern")
elif stem.endswith("facade"):
parts.append("Facade pattern")
elif stem.endswith("proxy"):
parts.append("Proxy pattern")
elif stem.endswith("strategy") or stem.endswith("strategies"):
parts.append("Strategy pattern")
elif stem.endswith("command") or stem.endswith("commands"):
parts.append("Command pattern")
elif stem.endswith("singleton"):
parts.append("Singleton pattern")
elif stem.endswith("composite"):
parts.append("Composite pattern")
# Fowler / DDD pattern stems
elif stem.endswith("aggregate") or stem.endswith("aggregates"):
parts.append("aggregate root")
elif stem.endswith("specification") or stem.endswith("specifications"):
parts.append("Specification pattern")
elif stem.endswith("plugin") or stem.endswith("plugins"):
parts.append("plugin")
elif stem.endswith("sdk"):
parts.append("SDK")
elif stem.endswith("api"):
parts.append("API client")
elif stem in ("index", "__init__"):
parts.append("module entry point")
elif stem in ("main", "app"):
parts.append("application entry point")
elif stem in ("setup", "bootstrap"):
parts.append("application bootstrap")
# --- Fuzzy stem fallback (distance <= 1, stems >= 4 chars) ---
_STEM_SUFFIX_MAP = [
("controller", "API controller"),
("service", "service"),
("repository", "repository"),
("gateway", "gateway"),
("registry", "registry"),
("agent", "agent"),
("store", "state store"),
("factory", "factory"),
("strategy", "Strategy pattern"),
("observer", "Observer pattern"),
("builder", "Builder pattern"),
("command", "Command pattern"),
("mediator", "Mediator pattern"),
("visitor", "Visitor pattern"),
("facade", "Facade pattern"),
("proxy", "Proxy pattern"),
("composite", "Composite pattern"),
("aggregate", "aggregate root"),
("specification", "Specification pattern"),
("adapter", "adapter"),
("provider", "provider"),
("processor", "processor"),
("handler", "event handler"),
("listener", "event listener"),
("middleware", "middleware"),
("interceptor", "interceptor"),
("scheduler", "scheduler"),
("worker", "worker"),
("dispatcher", "dispatcher"),
("subscriber", "subscriber"),
("resolver", "resolver"),
("manager", "manager"),
("connector", "connector"),
("client", "client"),
("policy", "policy"),
("plugin", "plugin"),
]
if not parts and len(stem) >= 5:
for sfx, purpose_label in _STEM_SUFFIX_MAP:
if _fuzzy_suffix_match(stem, sfx):
parts.append(purpose_label)
break
# --- Content-based heuristics (checked against first 4K) ---
head4k = content[:4096]
if not parts:
if re.search(r"export\s+(?:const\s+)?enum\s+", head4k):
parts.append("enum definitions")
elif re.search(r"export\s+(?:type|interface)\s+", head4k) and not re.search(
r"export\s+(?:class|function|const\s+\w+\s*=)", head4k
):
parts.append("type definitions")
elif re.search(r"^export\s+\{", head4k, re.MULTILINE) and not re.search(
r"^(?:class|function|const|let|var)\s", head4k, re.MULTILINE
):
parts.append("barrel re-exports")
elif re.search(
r"export\s+(?:default\s+)?(?:const\s+)?(?:config|CONFIG|configuration)\b",
head4k,
):
parts.append("configuration")
elif (
language == "python"
and re.search(r"^from\s+\.\w+\s+import\s+", head4k, re.MULTILINE)
and len(content) < 500
):
parts.append("package re-exports")
# --- Entity-kind heuristic: only type/interface/enum entities => type definitions ---
if not parts and entities:
kinds = {
(ent.get("kind", "") if isinstance(ent, dict) else ent.kind)
for ent in entities
}
if kinds and kinds <= {"interface", "type", "enum"}:
if kinds == {"enum"}:
parts.append("enum definitions")
else:
parts.append("type definitions")
return "; ".join(parts) if parts else "general module"
def detect_validation(content: str, language: str) -> bool:
if language == "python":
# Explicit validation frameworks
if re.search(
r"pydantic|BaseModel|validator|field_validator|Annotated\[|attrs|@attr",
content[:4096],
):
return True
# Typed dataclass schemas used as validation boundary (frozen=True or type-annotated)
if re.search(r"@dataclass", content[:4096]) and re.search(
r"frozen\s*=\s*True|def\s+__post_init__|:\s*(?:str|int|float|bool|list|dict|Optional)",
content[:4096],
):
return True
# TypedDict / NamedTuple schemas
if re.search(r"TypedDict|NamedTuple", content[:4096]):
return True
return False
if language == "php":
# Symfony Validator, Laravel FormRequest/Validator, Respect\Validation, webmozart/assert
return bool(
re.search(
r"->validate\(|Validator::|FormRequest|Constraint|Respect::|"
r"webmozart.*Assert|assert\(|Validation::|Validated",
content[:4096],
)
)
return bool(
re.search(
r"\.safeParse|\.parse\(|zod\.|z\.|Joi\.|yup\.|ajv|validateSchema",
content[:4096],
)
)
def path_to_uri(path: str) -> str:
"""Convert an absolute file path to a file:// URI."""
abs_path = os.path.abspath(path)
return "file://" + quote(abs_path, safe="/:")
def uri_to_path(uri: str) -> str:
"""Convert a file:// URI to an absolute file path."""
if uri.startswith("file://"):
return unquote(uri[7:])
return uri
def find_entity_column(content: str, entity_name: str, line_1based: int) -> int:
"""Find the 0-based column of *entity_name* on the given 1-based line."""
lines = content.split("\n")
if line_1based < 1 or line_1based > len(lines):
return 0
line_text = lines[line_1based - 1]
idx = line_text.find(entity_name)
return max(idx, 0)
# ---------------------------------------------------------------------------
# Python parser
# ---------------------------------------------------------------------------
def parse_python(path: str, content: str, repo: str, mtime: float) -> FileResult:
imports: list[str] = []
exports_abstract: list[str] = []
implements: list[str] = []
extends: list[str] = []
entities: list[Entity] = []
imported_names: list[ImportedName] = []
exported_names: list[str] = []
try:
tree = ast.parse(content, filename=path)
for node in ast.iter_child_nodes(tree):
# -- imports --
if isinstance(node, ast.Import):
for alias in node.names:
imports.append(alias.name)
local = alias.asname or alias.name.split(".")[-1]
imported_names.append(
ImportedName(
local_name=local,
original_name=alias.name,
module=alias.name,
)
)
elif isinstance(node, ast.ImportFrom):
mod = node.module or ""
if mod:
imports.append(mod)
for alias in node.names:
local = alias.asname or alias.name
imported_names.append(
ImportedName(
local_name=local,
original_name=alias.name,
module=mod,
)
)
# -- classes --
elif isinstance(node, ast.ClassDef):
methods = []
decorators = [_decorator_name(d) for d in node.decorator_list]
for item in node.body:
if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
methods.append(item.name)
bases_list = [_name_of(b) for b in node.bases if _name_of(b)]
entities.append(
Entity(
name=node.name,
kind="class",
line=node.lineno,
methods=methods,
decorators=decorators,
bases=bases_list,
)
)
exported_names.append(node.name)
extends.extend(bases_list)
for b in bases_list:
if b in ("ABC", "Protocol") or "Base" in b:
implements.append(b)
# -- functions --
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
decorators = [_decorator_name(d) for d in node.decorator_list]
entities.append(
Entity(
name=node.name,
kind="function",
line=node.lineno,
methods=[],
decorators=decorators,
bases=[],
)
)
exported_names.append(node.name)
# -- top-level assignments (constants, type aliases) --
elif isinstance(node, ast.Assign):
for target in node.targets:
if isinstance(target, ast.Name) and target.id.isupper():
exported_names.append(target.id)
except SyntaxError:
# Fallback: regex for imports only
for m in re.finditer(
r"^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w.]+))",
content,
re.MULTILINE,
):
imp = m.group(1) or m.group(2)
if imp:
imports.append(imp)
# Detect ABC / Protocol definitions
for m in PY_ABC_RE.finditer(content):
name = m.group(0).split("(")[0].split()[-1]
if name not in exports_abstract:
exports_abstract.append(name)
return FileResult(
path=path,
language="python",
repo=repo,
mtime=mtime,
content_hash=file_hash(path),
imports=imports,
exports_abstract=exports_abstract,
implements=implements,
extends=extends,
purpose=infer_purpose(path, content, "python", [e.to_dict() for e in entities]),
naming_convention=detect_naming(path),
has_validation=detect_validation(content, "python"),
entities=[e.to_dict() for e in entities],
imported_names=[n.to_dict() for n in imported_names],
exported_names=exported_names,
)
def _decorator_name(node: ast.expr) -> str:
if isinstance(node, ast.Name):
return node.id
if isinstance(node, ast.Attribute):
return node.attr
if isinstance(node, ast.Call):
return _decorator_name(node.func)
return ""
def _name_of(node: ast.expr) -> str:
if isinstance(node, ast.Name):
return node.id
if isinstance(node, ast.Attribute):
return node.attr
return ""
# ---------------------------------------------------------------------------
# TypeScript parser (tree-sitter)
# ---------------------------------------------------------------------------
def _ts_get_decorators(node) -> list[str]: # type: ignore[no-untyped-def]
"""Extract @Decorator names from a node's preceding decorator nodes."""
decorators = []
for child in node.children:
if child.type == "decorator":
# decorator -> @expression -- get the identifier
for dchild in child.children:
if dchild.type == "identifier":
decorators.append(dchild.text.decode("utf-8"))
elif dchild.type == "call_expression":
fn = dchild.child_by_field_name("function")
if fn:
decorators.append(fn.text.decode("utf-8").split(".")[-1])
return decorators
def _ts_get_heritage(node) -> tuple[list[str], list[str]]: # type: ignore[no-untyped-def]
"""Extract extends and implements from class_heritage."""
extends: list[str] = []
implements: list[str] = []
for child in node.children:
if child.type == "class_heritage":
for hchild in child.children:
if hchild.type == "extends_clause":
for t in hchild.children:
if t.type in ("type_identifier", "identifier"):
extends.append(t.text.decode("utf-8"))
elif hchild.type == "implements_clause":
for t in hchild.children:
if t.type in ("type_identifier", "identifier"):
implements.append(t.text.decode("utf-8"))
return extends, implements
def _ts_get_methods(node) -> list[str]: # type: ignore[no-untyped-def]
"""Extract method/property names from a class or interface body."""
methods = []
body = node.child_by_field_name("body")
if not body:
return methods
skip = {"constructor", "if", "for", "while", "switch", "return"}
for member in body.children:
if member.type in (
"method_definition",
"method_signature",
"public_field_definition",
"property_signature",
"abstract_method_signature",
):
name_node = member.child_by_field_name("name")
if name_node:
name = name_node.text.decode("utf-8")
if name not in skip:
methods.append(name)
return methods
# NestJS decorators that imply constructor-parameter dependency injection.
_NESTJS_INJECTION_DECOS: frozenset[str] = frozenset(
{
"Injectable",
"Controller",
"Module",
"Guard",
"Interceptor",
"Pipe",
"Resolver",
}
)
def _ts_get_constructor_injections(node) -> list[str]: # type: ignore[no-untyped-def]
"""Extract interface/type names from constructor parameter types and @Inject decorators.
Handles NestJS constructor injection patterns where typed constructor parameters
declare interface dependencies. Only PascalCase names are returned to avoid
tracking primitive or built-in types.
"""
injected: list[str] = []
body = node.child_by_field_name("body")
if not body:
return injected
for member in body.children:
if member.type != "method_definition":
continue
name_node = member.child_by_field_name("name")
if not name_node or not name_node.text:
continue
if name_node.text.decode("utf-8") != "constructor":
continue
params = member.child_by_field_name("parameters")
if not params:
continue
for param in params.children:
if param.type not in ("required_parameter", "optional_parameter"):
continue
# Check for @Inject(TOKEN) decorator on the parameter
for child in param.children:
if child.type == "decorator":
for dc in child.children:
if dc.type == "call_expression":
fn = dc.child_by_field_name("function")
if fn and fn.text and fn.text.decode("utf-8") == "Inject":
args = dc.child_by_field_name("arguments")
if args:
for arg in args.children:
if arg.type in (
"identifier",
"string",
"template_string",
):
text = arg.text
if text:
token = text.decode("utf-8").strip(
"'\"`"
)
if token and token[0].isupper():
injected.append(token)
# Check type annotation on the parameter
type_ann = param.child_by_field_name("type")
if type_ann:
for tc in type_ann.children:
if tc.type in ("type_identifier", "identifier"):
ttext = tc.text
if ttext:
tname = ttext.decode("utf-8")
if tname and tname[0].isupper():
injected.append(tname)
return injected
def _ts_get_obj_keys(node) -> list[str]: # type: ignore[no-untyped-def]
"""Extract top-level property names from an object literal."""
keys = []
for child in node.children:
if child.type in ("pair", "method_definition", "shorthand_property_identifier"):
if child.type == "shorthand_property_identifier":
keys.append(child.text.decode("utf-8"))
else:
key = child.child_by_field_name("key")
if key:
keys.append(key.text.decode("utf-8"))
return keys
def parse_typescript_treesitter(
path: str,
content: str,
repo: str,
mtime: float,
) -> FileResult:
"""Parse a TypeScript file using tree-sitter for precise entity extraction."""
lang = _TSX_LANG if path.endswith(".tsx") else _TS_LANG
parser = _TsParser(lang)
tree = parser.parse(content.encode("utf-8"))
root = tree.root_node
imports: list[str] = []
exports_abstract: list[str] = []
implements_list: list[str] = []
extends_list: list[str] = []
entities: list[Entity] = []
imported_names: list[ImportedName] = []
exported_names: list[str] = []
for node in root.children:
is_export = node.type == "export_statement"
target = node
if is_export:
# The actual declaration is a child of export_statement.
# Skip keywords, decorators, and punctuation to find the declaration.
for child in node.children:
if child.type in (
"export",
"default",
"comment",
"decorator",
"{",
"}",
",",
";",
):
continue
target = child
break
# -- Imports --
if node.type == "import_statement":
source_node = node.child_by_field_name("source")
if source_node:
specifier = source_node.text.decode("utf-8").strip("'\"")
imports.append(specifier)
# Extract named imports
for child in node.children:
if child.type == "import_clause":
for ic in child.children:
if ic.type == "identifier":
# default import
imported_names.append(
ImportedName(
local_name=ic.text.decode("utf-8"),
original_name="default",
module=specifier,
)
)
elif ic.type == "named_imports":
for spec in ic.children:
if spec.type == "import_specifier":
name_node = spec.child_by_field_name("name")
alias_node = spec.child_by_field_name("alias")
if name_node:
orig = name_node.text.decode("utf-8")
local = (
alias_node.text.decode("utf-8")
if alias_node
else orig
)
imported_names.append(
ImportedName(
local_name=local,
original_name=orig,
module=specifier,
)
)
# -- Class --
elif target.type == "class_declaration":
name_node = target.child_by_field_name("name")
if not name_node:
continue
name = name_node.text.decode("utf-8")
line = target.start_point[0] + 1
decorators = _ts_get_decorators(node if is_export else target)
ext, impl = _ts_get_heritage(target)
methods = _ts_get_methods(target)
extends_list.extend(ext)
implements_list.extend(impl)
# NestJS @Injectable/@Controller/@Module: constructor param types are
# interface dependencies that should create CALLS_API edges.
if _NESTJS_INJECTION_DECOS.intersection(decorators):
constructor_deps = _ts_get_constructor_injections(target)
implements_list.extend(constructor_deps)
# Check if abstract
is_abstract = any(c.type == "abstract" for c in target.children)
if is_abstract:
exports_abstract.append(name)
entities.append(
Entity(
name=name,
kind="class",
line=line,
methods=methods,
decorators=decorators,
bases=ext + impl,
)
)
exported_names.append(name)
# -- Interface --
elif target.type == "interface_declaration":
name_node = target.child_by_field_name("name")
if not name_node:
continue
name = name_node.text.decode("utf-8")
line = target.start_point[0] + 1
methods = _ts_get_methods(target)
exports_abstract.append(name)
entities.append(
Entity(
name=name,
kind="interface",
line=line,
methods=methods,
decorators=[],
bases=[],
)
)
exported_names.append(name)
# -- Type alias --
elif target.type == "type_alias_declaration":
name_node = target.child_by_field_name("name")
if not name_node:
continue
name = name_node.text.decode("utf-8")
line = target.start_point[0] + 1
entities.append(
Entity(
name=name,
kind="type",
line=line,
methods=[],
decorators=[],
bases=[],
)
)
exported_names.append(name)
# -- Enum --
elif target.type == "enum_declaration":
name_node = target.child_by_field_name("name")
if not name_node:
continue
name = name_node.text.decode("utf-8")
line = target.start_point[0] + 1
entities.append(
Entity(
name=name,
kind="enum",
line=line,
methods=[],
decorators=[],
bases=[],
)
)
exported_names.append(name)
# -- Function --
elif target.type == "function_declaration":
name_node = target.child_by_field_name("name")
if not name_node:
continue
name = name_node.text.decode("utf-8")
line = target.start_point[0] + 1
decorators = _ts_get_decorators(node if is_export else target)
entities.append(
Entity(
name=name,
kind="function",
line=line,
methods=[],
decorators=decorators,
bases=[],
)
)
exported_names.append(name)
# -- Lexical declaration (const/let) --
elif target.type == "lexical_declaration":
for declarator in target.children:
if declarator.type != "variable_declarator":
continue
name_node = declarator.child_by_field_name("name")
if not name_node or name_node.type != "identifier":
continue
name = name_node.text.decode("utf-8")
line = declarator.start_point[0] + 1
value = declarator.child_by_field_name("value")
if not value:
continue
# Object literal: extract keys as methods
if value.type == "object":
obj_keys = _ts_get_obj_keys(value)
entities.append(
Entity(
name=name,
kind="function",
line=line,
methods=obj_keys,
decorators=[],
bases=[],
)
)
# new Constructor(...)
elif value.type == "new_expression":
entities.append(
Entity(
name=name,
kind="function",
line=line,
methods=[],
decorators=[],
bases=[],
)
)
# Arrow function or call expression
elif value.type in ("arrow_function", "call_expression"):
entities.append(
Entity(
name=name,
kind="function",
line=line,
methods=[],
decorators=[],
bases=[],
)
)
# Other const (e.g., z.object, string literal, etc.)
else:
entities.append(
Entity(
name=name,
kind="function",
line=line,
methods=[],
decorators=[],
bases=[],
)
)
exported_names.append(name)
file_lang = "tsx" if path.endswith(".tsx") else "typescript"
return FileResult(
path=path,
language=file_lang,
repo=repo,
mtime=mtime,
content_hash=file_hash(path),
imports=imports,
exports_abstract=exports_abstract,
implements=implements_list,
extends=extends_list,
purpose=infer_purpose(
path, content, "typescript", [e.to_dict() for e in entities]
),
naming_convention=detect_naming(path),
has_validation=detect_validation(content, "typescript"),
entities=[e.to_dict() for e in entities],
imported_names=[n.to_dict() for n in imported_names],
exported_names=exported_names,
)
# ---------------------------------------------------------------------------
# Worker
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# PHP parser (tree-sitter, with regex fallback)
# ---------------------------------------------------------------------------
# PHP 8 attribute syntax: #[Attribute], #[Attribute(args)], #[Route('/path')]
_PHP_ATTRIBUTE_RE = re.compile(r"#\[\s*([\w\\]+)")
# PHP namespace declaration: namespace App\Service;
_PHP_NAMESPACE_RE = re.compile(r"^\s*namespace\s+([\w\\]+)\s*;", re.MULTILINE)
# PHP class/interface/trait/enum declarations with optional modifiers and base list.
_PHP_DECL_RE = re.compile(
r"^\s*(?:(abstract|final)\s+)?"
r"(class|interface|trait|enum)\s+(\w+)"
r"(?::\s*(?:int|string)\s*)?" # backed enum type: enum Status: string
r"(?:\s+extends\s+([\w\\,\s]+?))?"
r"(?:\s+implements\s+([\w\\,\s]+?))?"
r"\s*\{",
re.MULTILINE,
)
# PHP top-level function declarations: function name(...) { ... }
_PHP_FUNCTION_RE = re.compile(r"^\s*function\s+(\w+)\s*\(", re.MULTILINE)
# PHP use statements (imports): use Foo\Bar; use Foo\Bar as Baz; use Foo\Bar\{Qux, Quux};
_PHP_USE_RE = re.compile(
r"^\s*use\s+((?:function|const)?\s*[\w\\]+)"
r"(?:\s*\\\s*\{([^}]*)\})?"
r"(?:\s+as\s+(\w+))?\s*;",
re.MULTILINE,
)
# PHP class method signatures (simple, for method-name extraction).
_PHP_METHOD_RE = re.compile(
r"^\s*(?:(?:public|protected|private|static|final|abstract)\s+)*"
r"function\s+(\w+)\s*\(",
re.MULTILINE,
)
# PHP trait usage inside class body: use TraitA, TraitB;
_PHP_TRAIT_USE_RE = re.compile(r"^\s*use\s+([\w\\,\s]+?);", re.MULTILINE)
def _php_normalize_name(name: str) -> str:
"""Strip leading backslash and return short class name from an FQCN.
`App\\Service\\Foo` -> `Foo`. Bare names pass through.
"""
name = name.strip().lstrip("\\")
if "\\" in name:
return name.rsplit("\\", 1)[-1]
return name
def _php_split_names(csv: str) -> list[str]:
"""Split 'A, B, \\C\\D' into short names, dropping empties."""
out = []
for part in (csv or "").split(","):
part = part.strip()
if part:
out.append(_php_normalize_name(part))
return out
def parse_php_regex(path: str, content: str, repo: str, mtime: float) -> FileResult:
"""Fallback PHP parser using regex. Used when tree-sitter-php is unavailable.
Extracts namespaces, use statements, class/interface/trait/enum declarations
(with heritage and methods), and top-level functions. Less precise than the
tree-sitter parser: no PHP 8 attributes, no constructor DI, and method
extraction is best-effort. Still produces enough facts to make PHP files
visible in the graph.
"""
imports: list[str] = []
exported_names: list[str] = []
exported_abstract: list[str] = []
implements_list: list[str] = []
extends_list: list[str] = []
entities: list[Entity] = []
imported_names: list[ImportedName] = []
traits_used: list[str] = []
# Namespace
ns_match = _PHP_NAMESPACE_RE.search(content)
# Use statements (imports)
for m in _PHP_USE_RE.finditer(content):
spec = m.group(1).strip()
# Skip 'use function X' / 'use const X' prefixes for module import edges
is_func = spec.lower().startswith("function ")
is_const = spec.lower().startswith("const ")
if is_func or is_const:
spec = spec.split(None, 1)[1]
group = m.group(2)
alias = m.group(3)
if group:
# use Foo\Bar\{Baz, Qux};
base = spec.rstrip("\\")
for piece in group.split(","):
piece = piece.strip()
if not piece:
continue
fqcn = base + "\\" + piece if base else piece
local = alias or piece
imports.append(fqcn)
imported_names.append(
ImportedName(
local_name=local,
original_name=piece,
module=fqcn,
)
)
else:
short = _php_normalize_name(spec)
local = alias or short
imports.append(spec)
imported_names.append(
ImportedName(
local_name=local,
original_name=short,
module=spec,
)
)
# Declarations (class/interface/trait/enum)
for m in _PHP_DECL_RE.finditer(content):
modifier, kind_kw, name, extends_csv, implements_csv = m.groups()
line = content.count("\n", 0, m.start()) + 1
ext = _php_split_names(extends_csv)
impl = _php_split_names(implements_csv)
extends_list.extend(ext)
implements_list.extend(impl)
kind_map = {
"class": "class",
"interface": "interface",
"trait": "trait",
"enum": "enum",
}
kind = kind_map.get(kind_kw, "class")
# Extract methods: search the body bounded by this declaration. Without
# tree-sitter we cannot find the closing brace reliably, so we scan
# method signatures within a window (~next 500 lines).
body_start = m.end()
window = content[body_start : body_start + 20000]
methods = _PHP_METHOD_RE.findall(window)
# Trait usage inside class body
if kind == "class":
for tm in _PHP_TRAIT_USE_RE.finditer(window):
for tname in _php_split_names(tm.group(1)):
if tname not in traits_used:
traits_used.append(tname)
# Deduplicate preserving order
seen: set[str] = set()
methods_clean: list[str] = []
for meth in methods:
if meth not in seen:
seen.add(meth)
methods_clean.append(meth)
is_abstract = modifier == "abstract" or kind in ("interface", "trait")
if is_abstract:
exported_abstract.append(name)
entities.append(
Entity(
name=name,
kind=kind,
line=line,
methods=methods_clean,
decorators=[],
bases=ext + impl,
)
)
exported_names.append(name)
# Top-level functions (regex is coarse; may catch methods too -- dedupe by
# only accepting function names not already in entities).
decl_names = {e.name for e in entities}
for m in _PHP_FUNCTION_RE.finditer(content):
fname = m.group(1)
if fname in decl_names:
continue
line = content.count("\n", 0, m.start()) + 1
entities.append(
Entity(
name=fname,
kind="function",
line=line,
methods=[],
decorators=[],
bases=[],
)
)
exported_names.append(fname)
decl_names.add(fname)
return FileResult(
path=path,
language="php",
repo=repo,
mtime=mtime,
content_hash=file_hash(path),
imports=imports,
exports_abstract=exported_abstract,
implements=implements_list,
extends=extends_list,
purpose=infer_purpose(path, content, "php", [e.to_dict() for e in entities]),
naming_convention=detect_naming(path),
has_validation=detect_validation(content, "php"),
entities=[e.to_dict() for e in entities],
imported_names=[n.to_dict() for n in imported_names],
exported_names=exported_names,
traits_used=traits_used,
)
# ---------------------------------------------------------------------------
# tree-sitter PHP parser helpers
# ---------------------------------------------------------------------------
def _php_text(node) -> str: # type: ignore[no-untyped-def]
"""Decode a tree-sitter node's text, or return '' if the node is None."""
return node.text.decode("utf-8") if node and node.text else ""
def _php_short(text: str) -> str:
r"""`App\Service\Foo` -> `Foo`, strip leading backslash. Bare names pass through."""
text = text.strip().lstrip("\\")
if "\\" in text:
return text.rsplit("\\", 1)[-1]
return text
def _php_get_attributes(node) -> list[str]: # type: ignore[no-untyped-def]
"""Extract PHP 8 #[Attribute] names from attribute_list siblings of a declaration."""
decos: list[str] = []
for child in node.children:
if child.type != "attribute_list":
continue
for group in child.children:
if group.type == "attribute_group":
for attr in group.children:
if attr.type == "attribute":
name_node = attr.child_by_field_name("name")
if name_node:
decos.append(_php_short(_php_text(name_node)))
else:
# Fallback: first name/identifier child
for c in attr.children:
if c.type in ("name", "qualified_name"):
decos.append(_php_short(_php_text(c)))
break
return decos
def _php_collect_qualified_names(node) -> list[str]: # type: ignore[no-untyped-def]
"""Collect short names from all qualified_name/name children of `node`."""
out: list[str] = []
for child in node.children:
if child.type in ("qualified_name", "name"):
out.append(_php_short(_php_text(child)))
return out
def _php_get_heritage(node) -> tuple[list[str], list[str]]: # type: ignore[no-untyped-def]
"""Extract (extends, implements) short names from base_clause / class_interface_clause."""
extends: list[str] = []
implements: list[str] = []
for child in node.children:
if child.type == "base_clause":
extends.extend(_php_collect_qualified_names(child))
elif child.type == "class_interface_clause":
implements.extend(_php_collect_qualified_names(child))
return extends, implements
def _php_get_methods(body_node) -> list[str]: # type: ignore[no-untyped-def]
"""Extract method names from a declaration_list / enum_declaration_list body."""
methods: list[str] = []
if not body_node:
return methods
skip = {
"__construct",
"__destruct",
"__get",
"__set",
"__call",
"__isset",
"__unset",
"__toString",
"__invoke",
"__clone",
"__serialize",
"__unserialize",
"__sleep",
"__wakeup",
"__set_state",
}
for member in body_node.children:
if member.type == "method_declaration":
name_node = member.child_by_field_name("name")
if name_node:
name = _php_text(name_node)
if name and name not in skip:
methods.append(name)
return methods
def _php_get_trait_uses(body_node) -> list[str]: # type: ignore[no-untyped-def]
"""Extract trait names from `use TraitA, TraitB;` inside a class body."""
traits: list[str] = []
if not body_node:
return traits
for member in body_node.children:
if member.type != "use_declaration":
continue
for child in member.children:
if child.type in ("qualified_name", "name"):
traits.append(_php_short(_php_text(child)))
elif child.type == "namespace_name":
# A bare namespace_name (uncommon in trait use)
traits.append(_php_short(_php_text(child)))
return traits
def _php_get_constructor_di(node) -> list[str]: # type: ignore[no-untyped-def]
"""Extract PascalCase type-hints from constructor parameters (Symfony/Laravel DI).
Mirrors `_ts_get_constructor_injections`: typed constructor params declare
service dependencies. Only PascalCase names are returned (primitives filtered).
"""
injected: list[str] = []
body = node.child_by_field_name("body")
if not body:
return injected
for member in body.children:
if member.type != "method_declaration":
continue
name_node = member.child_by_field_name("name")
if not name_node or _php_text(name_node) != "__construct":
continue
params = member.child_by_field_name("parameters")
if not params:
continue
for param in params.children:
if param.type not in (
"simple_parameter",
"property_promotion_parameter",
"variadic_parameter",
):
continue
# Type node may be: named_type, primitive_type, optional_type, or null.
type_node = None
for c in param.children:
if c.type in ("named_type", "optional_type"):
type_node = c
break
if not type_node:
continue
# optional_type wraps named_type; named_type wraps name/qualified_name.
short = ""
for tc in type_node.children:
if tc.type in ("named_type", "qualified_name", "name"):
txt = _php_text(tc)
if txt:
short = _php_short(txt)
break
if short and short[:1].isupper():
injected.append(short)
return injected
def parse_php_treesitter(
path: str,
content: str,
repo: str,
mtime: float,
) -> FileResult:
"""Parse a PHP file using tree-sitter for precise entity extraction.
Handles classes, interfaces, traits, PHP 8.1 enums, top-level functions,
PHP 8 attributes (#[...]), heritage (extends/implements), trait usage
inside class bodies, and Symfony/Laravel constructor dependency injection.
"""
parser = _TsParser(_PHP_LANG)
tree = parser.parse(content.encode("utf-8"))
root = tree.root_node
imports: list[str] = []
exported_abstract: list[str] = []
implements_list: list[str] = []
extends_list: list[str] = []
entities: list[Entity] = []
imported_names: list[ImportedName] = []
exported_names: list[str] = []
traits_used: list[str] = []
for node in root.children:
# -- Namespace declaration --
if node.type == "namespace_definition":
continue # tracked via use-resolution; not an entity
# -- Use declarations (imports) --
if node.type == "namespace_use_declaration":
# Check for `use function X` / `use const X` modifier
kind = "class"
for child in node.children:
if child.type in ("function", "const") and _php_text(child) in (
"function",
"const",
):
kind = _php_text(child)
# The namespace_use_clause holds the qualified_name + optional alias.
for child in node.children:
if child.type != "namespace_use_clause":
continue
clause = child
qname = ""
alias = ""
for cc in clause.children:
if cc.type in ("qualified_name", "name") and not qname:
qname = _php_text(cc)
elif cc.type == "namespace_name" and not qname:
qname = _php_text(cc)
elif cc.type == "name" and clause.children[0].type in ("as",):
alias = _php_text(cc)
# Alias: look for `as NAME`
for cc in clause.children:
if cc.type == "as":
idx = list(clause.children).index(cc)
if idx + 1 < len(clause.children):
alias = _php_text(clause.children[idx + 1])
if not qname:
continue
short = _php_short(qname)
local = alias or short
imports.append(qname)
imported_names.append(
ImportedName(
local_name=local,
original_name=short,
module=qname,
)
)
continue
# -- Class / Interface / Trait / Enum declarations --
decl_kind = {
"class_declaration": "class",
"interface_declaration": "interface",
"trait_declaration": "trait",
"enum_declaration": "enum",
}.get(node.type)
if decl_kind is None:
# -- Top-level function --
if node.type == "function_definition":
name_node = node.child_by_field_name("name")
if not name_node:
continue
name = _php_text(name_node)
entities.append(
Entity(
name=name,
kind="function",
line=node.start_point[0] + 1,
methods=[],
decorators=_php_get_attributes(node),
bases=[],
)
)
exported_names.append(name)
continue
name_node = node.child_by_field_name("name")
if not name_node:
continue
name = _php_text(name_node)
line = node.start_point[0] + 1
decorators = _php_get_attributes(node)
ext, impl = _php_get_heritage(node)
# Body node differs: declaration_list (class/interface/trait) vs enum_declaration_list.
body_node = None
for child in node.children:
if child.type in ("declaration_list", "enum_declaration_list"):
body_node = child
break
methods = _php_get_methods(body_node)
extends_list.extend(ext)
implements_list.extend(impl)
# Trait usage inside class body -> CALLS_API edges (Phase 4).
class_traits = _php_get_trait_uses(body_node)
traits_used.extend(class_traits)
# Abstract modifier for classes; interfaces/traits/enums are inherently abstract.
is_abstract = any(
c.type == "abstract_modifier" for c in node.children
) or decl_kind in ("interface", "trait")
if is_abstract:
exported_abstract.append(name)
# Constructor DI: collect type-hints from __construct params. Triggered for
# any class with a constructor (Symfony/Laravel auto-wiring) -- matches the
# NestJS heuristic in the TS parser.
if decl_kind == "class":
implements_list.extend(_php_get_constructor_di(node))
entities.append(
Entity(
name=name,
kind=decl_kind,
line=line,
methods=methods,
decorators=decorators,
bases=ext + impl,
)
)
exported_names.append(name)
return FileResult(
path=path,
language="php",
repo=repo,
mtime=mtime,
content_hash=file_hash(path),
imports=imports,
exports_abstract=exported_abstract,
implements=implements_list,
extends=extends_list,
purpose=infer_purpose(path, content, "php", [e.to_dict() for e in entities]),
naming_convention=detect_naming(path),
has_validation=detect_validation(content, "php"),
entities=[e.to_dict() for e in entities],
imported_names=[n.to_dict() for n in imported_names],
exported_names=exported_names,
traits_used=traits_used,
)
def parse_file(args: tuple[str, str, str]) -> Optional[FileResult]:
path, language, repo = args
try:
mtime = os.path.getmtime(path)
with open(path, "r", encoding="utf-8", errors="replace") as f:
content = f.read()
if language == "python":
return parse_python(path, content, repo, mtime)
if language == "php":
# tree-sitter-php if available, regex fallback otherwise.
if _PHP_LANG is not None:
try:
return parse_php_treesitter(path, content, repo, mtime)
except Exception:
return parse_php_regex(path, content, repo, mtime)
return parse_php_regex(path, content, repo, mtime)
return parse_typescript_treesitter(path, content, repo, mtime)
except Exception:
return None
# ---------------------------------------------------------------------------
# File discovery
# ---------------------------------------------------------------------------
def discover_files(root: str) -> list[tuple[str, str, str]]:
results: list[tuple[str, str, str]] = []
root_path = Path(root).resolve()
def _walk(dir_path: Path, repo: str):
try:
with os.scandir(dir_path) as it:
for entry in it:
if entry.name in SKIP_DIRS:
continue
if entry.is_dir(follow_symlinks=False):
child_repo = repo
if (
entry.name != ".git"
and (dir_path / entry.name / ".git").exists()
):
child_repo = entry.name
_walk(dir_path / entry.name, child_repo)
elif entry.is_file(follow_symlinks=False):
name = entry.name
if name.endswith(".py"):
results.append((entry.path, "python", repo))
elif (
name.endswith(".ts") or name.endswith(".tsx")
) and not name.endswith(".d.ts"):
results.append((entry.path, "typescript", repo))
elif name.endswith(".php") and not name.endswith(".blade.php"):
# Blade templates (.blade.php) are Laravel's HTML
# templating layer -- skip them like .d.ts.
results.append((entry.path, "php", repo))
except PermissionError:
pass
_walk(root_path, root_path.name)
return results
# ---------------------------------------------------------------------------
# Incremental cache
# ---------------------------------------------------------------------------
def load_cache(root: str) -> dict:
cache_path = os.path.join(root, CACHE_FILE)
if os.path.exists(cache_path):
try:
with open(cache_path, "r") as f:
return json.load(f)
except (json.JSONDecodeError, OSError):
pass
return {}
def save_cache(root: str, cache: dict):
cache_path = os.path.join(root, CACHE_FILE)
with open(cache_path, "w") as f:
json.dump(cache, f)
# ---------------------------------------------------------------------------
# Import resolution
# ---------------------------------------------------------------------------
# TypeScript path alias resolution (tsconfig.json "paths")
# ---------------------------------------------------------------------------
def _load_ts_path_aliases(root: str) -> dict[str, str]:
"""Scan for tsconfig.json files and build a global alias -> absolute directory map.
For each tsconfig with compilerOptions.paths, resolves alias patterns like
``@sdk/* -> ['../../sdk/src/*']`` relative to the tsconfig's directory.
Returns {alias_prefix: absolute_directory} e.g. {"@sdk/": "/abs/path/to/sdk/src/"}.
"""
import json as _json
alias_map: dict[str, str] = {}
# Walk up to 4 directory levels for tsconfig files (covers monorepo structures)
for dirpath, dirnames, filenames in os.walk(root):
# Skip node_modules, .git, dist, build
dirnames[:] = [
d
for d in dirnames
if d not in ("node_modules", ".git", "dist", "build", ".next", ".cache")
]
depth = dirpath[len(root) :].count(os.sep)
if depth > 5:
dirnames.clear()
continue
for fname in filenames:
if fname == "tsconfig.json":
tsconfig_path = os.path.join(dirpath, fname)
try:
with open(tsconfig_path, "r") as f:
raw = f.read()
# Strip single-line comments (tsconfig allows them)
raw = re.sub(r"//.*$", "", raw, flags=re.MULTILINE)
# Strip trailing commas before } or ]
raw = re.sub(r",(\s*[}\]])", r"\1", raw)
data = _json.loads(raw)
paths = data.get("compilerOptions", {}).get("paths", {})
for alias_pattern, targets in paths.items():
if not alias_pattern.endswith("/*") or not targets:
continue
prefix = alias_pattern[:-1] # "@sdk/*" -> "@sdk/"
target = targets[0] # Take first target
if not target.endswith("/*"):
continue
target_dir = target[
:-2
] # "./*" -> ".", "../../sdk/src/*" -> "../../sdk/src"
abs_target = os.path.normpath(os.path.join(dirpath, target_dir))
# If target dir has a `src/` subdirectory, prefer it
# (common monorepo pattern: @sdk/* -> ['./*'] maps to pkg root,
# but actual source lives in pkg/src/)
src_subdir = os.path.join(abs_target, "src")
effective_target = (
src_subdir if os.path.isdir(src_subdir) else abs_target
)
if os.path.isdir(abs_target):
# Longer (more specific) prefixes take priority
if prefix not in alias_map or len(effective_target) > len(
alias_map[prefix]
):
alias_map[prefix] = effective_target
except Exception:
continue
return alias_map
def _resolve_ts_path_alias(
imp: str,
source_path: str,
root: str,
path_to_id: dict[str, str],
alias_map: dict[str, str],
) -> Optional[str]:
"""Resolve a TypeScript import like '@sdk/llm/llmClientApi' using tsconfig paths.
Tries the most specific alias prefix first (longest match).
"""
if not alias_map:
return None
# Sort by prefix length descending so "@agent-upgraide/" matches before "@a/"
for prefix in sorted(alias_map, key=len, reverse=True):
if imp.startswith(prefix):
remainder = imp[len(prefix) :]
base_dir = alias_map[prefix]
resolved = os.path.normpath(os.path.join(base_dir, remainder))
for ext in (
"",
".ts",
".tsx",
"/index.ts",
"/index.tsx",
".js",
"/index.js",
):
candidate = resolved + ext
if candidate in path_to_id:
return path_to_id[candidate]
# path_to_id may use relative paths -- try relative form too
try:
rel_candidate = (
os.path.relpath(candidate, root) if root else candidate
)
except ValueError:
rel_candidate = candidate
if rel_candidate in path_to_id:
return path_to_id[rel_candidate]
return None
# ---------------------------------------------------------------------------
def _load_composer_autoload(root: str) -> dict[str, str]:
"""Scan for composer.json files and build a PSR-4 namespace -> directory map.
For each composer.json with ``autoload.psr-4`` (and ``autoload-dev.psr-4``),
resolves namespace prefixes like ``"App\\\\": "src/"`` to absolute directories.
Returns ``{namespace_prefix: absolute_directory}`` e.g.
``{"App\\\\": "/abs/path/to/src/"}``.
"""
import json as _json
psr4_map: dict[str, str] = {}
for dirpath, dirnames, filenames in os.walk(root):
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
depth = dirpath[len(root) :].count(os.sep)
if depth > 5:
dirnames.clear()
continue
for fname in filenames:
if fname != "composer.json":
continue
composer_path = os.path.join(dirpath, fname)
try:
with open(composer_path, "r") as f:
data = _json.load(f)
except (ValueError, OSError):
continue
autoload = data.get("autoload", {})
autoload_dev = data.get("autoload-dev", {})
for section in (autoload, autoload_dev):
psr4 = section.get("psr-4", {})
for ns_prefix, paths in psr4.items():
if not ns_prefix:
continue
# Normalize: ensure trailing backslash
ns = ns_prefix if ns_prefix.endswith("\\") else ns_prefix + "\\"
# paths may be a string or list of strings
if isinstance(paths, str):
paths = [paths]
for rel_dir in paths:
abs_dir = os.path.normpath(os.path.join(dirpath, rel_dir))
if os.path.isdir(abs_dir):
psr4_map[ns] = abs_dir
return psr4_map
def _resolve_php_psr4(
fqcn: str,
path_to_id: dict[str, str],
psr4_map: dict[str, str],
) -> Optional[str]:
"""Resolve a PHP FQCN to a module ID via PSR-4 autoload rules.
Tries the longest matching namespace prefix first (e.g. ``App\\\\Service\\\\``
before ``App\\\\``), replaces the prefix with the mapped directory, converts
remaining namespace segments to path separators, and appends ``.php``.
"""
fqcn = fqcn.strip().lstrip("\\")
if not psr4_map or not fqcn:
return None
# Sort prefixes by length descending for longest-match-first
for ns_prefix in sorted(psr4_map, key=len, reverse=True):
if fqcn.startswith(ns_prefix):
remainder = fqcn[len(ns_prefix) :]
base_dir = psr4_map[ns_prefix]
# PSR-4: remainder maps directly to filesystem path
rel_path = remainder.replace("\\", os.sep) + ".php"
candidate = os.path.normpath(os.path.join(base_dir, rel_path))
if candidate in path_to_id:
return path_to_id[candidate]
return None
def resolve_import(
imp: str,
source_path: str,
root: str,
path_to_id: dict[str, str],
language: str,
ts_path_aliases: "dict[str, str] | None" = None,
php_psr4: "dict[str, str] | None" = None,
) -> Optional[str]:
if language == "typescript":
if not imp.startswith(".") and not imp.startswith("/"):
# Try tsconfig paths alias resolution before giving up
resolved_via_alias = _resolve_ts_path_alias(
imp,
source_path,
root,
path_to_id,
ts_path_aliases or {},
)
if resolved_via_alias:
return resolved_via_alias
return None
source_dir = os.path.dirname(source_path)
resolved = os.path.normpath(os.path.join(source_dir, imp))
for ext in ("", ".ts", ".tsx", "/index.ts", "/index.tsx", ".js", "/index.js"):
candidate = resolved + ext
if candidate in path_to_id:
return path_to_id[candidate]
elif language == "python":
parts = imp.split(".")
# Build candidate base directories: walk up from source_dir to root,
# collecting every directory that contains the first import segment.
bases: list[str] = [root, os.path.dirname(source_path)]
first_segment = parts[0] if parts else ""
cur = os.path.dirname(source_path)
while cur and cur >= root:
if os.path.isdir(os.path.join(cur, first_segment)):
if cur not in bases:
bases.append(cur)
parent = os.path.dirname(cur)
if parent == cur:
break
cur = parent
for base in bases:
# Try direct .py file first, then __init__.py
for ext in (".py", "/__init__.py"):
if ext == ".py":
candidate = os.path.join(base, *parts) + ext
else:
candidate = os.path.join(base, *parts, "__init__.py")
candidate = os.path.normpath(candidate)
if candidate in path_to_id:
return path_to_id[candidate]
# Gap 4 fix: "from modules.commons import json_doc_sqlite"
# The AST records module="modules.commons", name="json_doc_sqlite".
# Check if the imported names include a sub-module file.
if len(parts) >= 2:
pkg_dir = os.path.join(base, *parts)
if os.path.isdir(pkg_dir):
# This is a package -- imported names may be sub-modules.
# resolve_import is called per import specifier (module level),
# so we already resolved to __init__.py above. The sub-module
# resolution happens in _resolve_submodule_imports below.
pass
elif language == "php":
# PSR-4 autoload resolution: FQCN -> file path via composer.json rules.
resolved_via_psr4 = _resolve_php_psr4(imp, path_to_id, php_psr4 or {})
if resolved_via_psr4:
return resolved_via_psr4
# Fallback: walk-up heuristic when no composer.json or no match.
# Convert namespace separators to path separators and try each base.
parts = [p for p in imp.replace("\\", "/").split("/") if p]
if not parts:
return None
bases: list[str] = [root, os.path.dirname(source_path)]
first_segment = parts[0]
cur = os.path.dirname(source_path)
while cur and cur >= root:
if os.path.isdir(os.path.join(cur, first_segment)):
if cur not in bases:
bases.append(cur)
parent = os.path.dirname(cur)
if parent == cur:
break
cur = parent
for base in bases:
candidate = os.path.normpath(os.path.join(base, *parts) + ".php")
if candidate in path_to_id:
return path_to_id[candidate]
return None
def _resolve_submodule_imports(
result: "FileResult",
root: str,
path_to_id: dict[str, str],
) -> list[str]:
"""
Fix Python 'from package import submodule' patterns.
When `from modules.commons import json_doc_sqlite` is parsed, the AST
records module='modules.commons' and name='json_doc_sqlite'. If
json_doc_sqlite.py exists as a sibling file, add it as an extra import.
"""
extra_imports: list[str] = []
if result.language != "python":
return extra_imports
# Build base dirs: walk up from source to root (same as resolve_import)
all_bases: list[str] = [root, os.path.dirname(result.path)]
for imp_name in result.imported_names:
mod = (
imp_name.get("module", "")
if isinstance(imp_name, dict)
else imp_name.module
)
orig = (
imp_name.get("original", "")
if isinstance(imp_name, dict)
else imp_name.original_name
)
if not mod or not orig:
continue
parts = mod.split(".")
first_seg = parts[0] if parts else ""
# Collect all viable bases including walk-up
bases = list(all_bases)
cur = os.path.dirname(result.path)
while cur and cur >= root:
if os.path.isdir(os.path.join(cur, first_seg)):
if cur not in bases:
bases.append(cur)
parent = os.path.dirname(cur)
if parent == cur:
break
cur = parent
for base in bases:
submod_path = os.path.normpath(os.path.join(base, *parts, orig + ".py"))
if submod_path in path_to_id:
target_id = path_to_id[submod_path]
if target_id not in extra_imports:
extra_imports.append(target_id)
break
return extra_imports
def make_node_id(path: str, root: str) -> str:
rel = os.path.relpath(path, root).replace(os.sep, "/")
for ext in (".tsx", ".ts", ".py", ".php"):
if rel.endswith(ext):
rel = rel[: -len(ext)]
break
if rel.endswith("/index"):
rel = rel[: -len("/index")]
return rel
# ---------------------------------------------------------------------------
# Graph building
# ---------------------------------------------------------------------------
def build_graph(
results: list[FileResult],
root: str,
) -> tuple[list[dict], list[dict]]:
path_to_id: dict[str, str] = {}
id_to_result: dict[str, FileResult] = {}
for r in results:
node_id = make_node_id(r.path, root)
path_to_id[r.path] = node_id
id_to_result[node_id] = r
# Load tsconfig.json path aliases for resolving @sdk/*, @agent-*/* etc.
ts_path_aliases = _load_ts_path_aliases(root)
if ts_path_aliases:
print(
f"[belief-map] Loaded {len(ts_path_aliases)} TS path aliases",
file=sys.stderr,
)
# Load composer.json PSR-4 autoload rules for PHP namespace resolution.
php_psr4 = _load_composer_autoload(root)
if php_psr4:
print(
f"[belief-map] Loaded {len(php_psr4)} PHP PSR-4 namespaces",
file=sys.stderr,
)
# Map: abstract/interface name -> defining node id
abstract_providers: dict[str, str] = {}
for nid, r in id_to_result.items():
for ab in r.exports_abstract:
abstract_providers[ab] = nid
# Also register ALL interface-kind entities regardless of whether they are in
# exports_abstract (which only captures explicit exports). Non-exported interfaces
# used locally and interfaces from imported modules are resolved by bare name at
# the CALLS_API detection step, so they must be present in this map.
for nid, r in id_to_result.items():
for ent in r.entities:
ent_kind = (
ent.get("kind", "")
if isinstance(ent, dict)
else getattr(ent, "kind", "")
)
ent_name = (
ent.get("name", "")
if isinstance(ent, dict)
else getattr(ent, "name", "")
)
if (
ent_kind in ("interface", "trait")
and ent_name
and ent_name not in abstract_providers
):
abstract_providers[ent_name] = nid
# Map: (node_id, entity_name) -> True (for cross-file entity lookups)
entity_index: dict[str, str] = {} # "EntityName" -> "node_id"
for nid, r in id_to_result.items():
for name in r.exported_names:
# First definition wins (avoids overwrites from re-exports)
if name not in entity_index:
entity_index[name] = nid
# Register any entity that is used as a base class by other classes.
# This catches BaseModel, CanActivate, BaseProcessor, BaseAgent, etc.
# that are defined in the codebase but not marked as abstract/interface.
all_bases: set[str] = set()
for r in results:
for base in r.implements + r.extends:
all_bases.add(base)
for base_name in all_bases:
if base_name not in abstract_providers and base_name in entity_index:
abstract_providers[base_name] = entity_index[base_name]
nodes: list[dict] = []
edges: list[dict] = []
seen_edges: set[tuple] = set()
for node_id, r in id_to_result.items():
rel_path = os.path.relpath(r.path, root)
# Gap 3 fix: enrich purpose with entity-name heuristics
purpose = r.purpose
if r.entities and purpose in (
"general module",
"NestJS wiring module",
"NestJS injectable",
):
enhanced = infer_purpose(r.path, "", r.language, r.entities)
if enhanced not in (
"general module",
"NestJS wiring module",
"NestJS injectable",
):
purpose = enhanced
# -- build node with entities --
node: dict = {
"id": node_id,
"language": r.language,
"path": rel_path,
"repo": r.repo,
"invariant": {
"naming": r.naming_convention,
"package": r.repo,
},
"purpose": purpose,
}
if r.entities:
node["entities"] = r.entities
nodes.append(node)
# -- Gap 4: sub-module Python imports --
extra_submod_ids = _resolve_submodule_imports(r, root, path_to_id)
for target_id in extra_submod_ids:
if target_id != node_id:
edge_key = (node_id, target_id, "IMPORTS")
if edge_key not in seen_edges:
seen_edges.add(edge_key)
target_r = id_to_result.get(target_id)
via_base = bool(target_r and target_r.exports_abstract)
edges.append(
{
"source": node_id,
"target": target_id,
"type": "IMPORTS",
"via_base": via_base,
}
)
# -- module-level IMPORTS edges --
for imp in r.imports:
target_id = resolve_import(
imp, r.path, root, path_to_id, r.language, ts_path_aliases, php_psr4
)
if target_id and target_id != node_id:
edge_key = (node_id, target_id, "IMPORTS")
if edge_key not in seen_edges:
seen_edges.add(edge_key)
target_r = id_to_result.get(target_id)
via_base = bool(target_r and target_r.exports_abstract)
edges.append(
{
"source": node_id,
"target": target_id,
"type": "IMPORTS",
"via_base": via_base,
}
)
# -- CALLS_API edges (implements/extends + entity :bases) --
# Source 1: FileResult.implements + extends (from heritage clauses)
calls_api_targets: set[str] = set()
for iface in r.implements + r.extends:
provider_id = abstract_providers.get(iface)
if provider_id and provider_id != node_id:
calls_api_targets.add(provider_id)
# Source 2: Entity :bases cross-reference (catches bases not in
# heritage clauses, and bases defined in other modules)
for ent in r.entities:
for base_name in ent.get("bases", []):
provider_id = abstract_providers.get(base_name)
if provider_id and provider_id != node_id:
calls_api_targets.add(provider_id)
# Source 3: PHP trait usage (use TraitX; inside class body).
# Traits are abstract providers -- classes that use them should have
# CALLS_API edges to the trait's defining module.
for trait_name in r.traits_used:
provider_id = abstract_providers.get(trait_name)
if provider_id and provider_id != node_id:
calls_api_targets.add(provider_id)
for provider_id in calls_api_targets:
edge_key = (node_id, provider_id, "CALLS_API")
if edge_key not in seen_edges:
seen_edges.add(edge_key)
edges.append(
{
"source": node_id,
"target": provider_id,
"type": "CALLS_API",
"via_interface": True,
}
)
# -- DATA_FLOWS_TO edges --
for imp in r.imports:
target_id = resolve_import(
imp, r.path, root, path_to_id, r.language, ts_path_aliases, php_psr4
)
if target_id and target_id != node_id:
target_r = id_to_result.get(target_id)
if target_r and target_r.has_validation:
edge_key = (node_id, target_id, "DATA_FLOWS_TO")
if edge_key not in seen_edges:
seen_edges.add(edge_key)
edges.append(
{
"source": node_id,
"target": target_id,
"type": "DATA_FLOWS_TO",
"validated": True,
}
)
# -- entity-level REFERENCES edges --
# For each imported name, link source_entity -> target_entity
for imp_name in r.imported_names:
target_id = resolve_import(
imp_name["module"],
r.path,
root,
path_to_id,
r.language,
ts_path_aliases,
php_psr4,
)
if not target_id or target_id == node_id:
continue
target_r = id_to_result.get(target_id)
if not target_r:
continue
orig = imp_name["original"]
local = imp_name["local"]
# Find matching entity in target
target_entity = None
for ent in target_r.entities:
if ent["name"] == orig or (orig == "default" and ent["name"] == local):
target_entity = ent["name"]
break
if not target_entity:
# Still useful: the name is exported even if not a class/function entity
if orig in target_r.exported_names:
target_entity = orig
elif orig == "default" and target_r.exported_names:
target_entity = target_r.exported_names[0]
else:
continue
# Find which local entities reference this imported name
referencing_entities = _find_local_references(r.entities, local)
if referencing_entities:
for ref_ent in referencing_entities:
edge_key = (
f"{node_id}::{ref_ent}",
f"{target_id}::{target_entity}",
"REFERENCES",
)
if edge_key not in seen_edges:
seen_edges.add(edge_key)
edges.append(
{
"source": f"{node_id}::{ref_ent}",
"target": f"{target_id}::{target_entity}",
"type": "REFERENCES",
}
)
else:
# Module-level reference (not inside a specific entity)
edge_key = (node_id, f"{target_id}::{target_entity}", "REFERENCES")
if edge_key not in seen_edges:
seen_edges.add(edge_key)
edges.append(
{
"source": node_id,
"target": f"{target_id}::{target_entity}",
"type": "REFERENCES",
}
)
# -- Gap 2: BOUNDARY violation detection --
# -- HTTP_CALLS edges (cross-service HTTP client detection) --
# Detect modules with entities named *Client, *ClientService, *ClientApi,
# *External, *Remote and infer the target service from the class name.
_HTTP_CLIENT_SUFFIXES = (
"Client",
"ClientService",
"ClientApi",
"External",
"Remote",
)
_REPO_NAMES = {r.repo for r in results}
for node_id, r in id_to_result.items():
for ent in r.entities:
ent_name = ent.get("name", "")
if not any(ent_name.endswith(sfx) for sfx in _HTTP_CLIENT_SUFFIXES):
continue
# Extract target service name from class name:
# AgentExternalClient -> agent, McpHubClientService -> mcp-hub
# LLMClientRemote -> llm
stem = ent_name
for sfx in sorted(_HTTP_CLIENT_SUFFIXES, key=len, reverse=True):
if stem.endswith(sfx):
stem = stem[: -len(sfx)]
break
if not stem:
continue
# Convert PascalCase to kebab-case for repo matching
kebab = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", stem).lower()
# Try to match against known repo names
target_repo = None
for repo in _REPO_NAMES:
if repo == r.repo:
continue # skip self
repo_lower = repo.lower()
if (
kebab == repo_lower
or kebab.startswith(repo_lower)
or repo_lower.startswith(kebab)
):
target_repo = repo
break
if target_repo:
edge_key = (node_id, target_repo, "HTTP_CALLS")
if edge_key not in seen_edges:
seen_edges.add(edge_key)
edges.append(
{
"source": node_id,
"target": target_repo,
"type": "HTTP_CALLS",
"transport": "http",
"client_entity": ent_name,
}
)
# Domain modules should not import directly from infrastructure without
# going through an interface (via_base). Flag violations.
for edge in edges:
if edge["type"] != "IMPORTS":
continue
src = edge["source"]
tgt = edge["target"]
src_purpose = id_to_result.get(src, None)
tgt_purpose = id_to_result.get(tgt, None)
if not src_purpose or not tgt_purpose:
continue
src_p = (src_purpose.purpose or "").lower()
tgt_p = (tgt_purpose.purpose or "").lower()
# Domain importing infrastructure without going through base class
if "domain" in src_p and "infrastructure" in tgt_p and not edge.get("via_base"):
edges.append(
{
"source": src,
"target": tgt,
"type": "VIOLATION",
"rule": "boundary",
"detail": "domain imports infrastructure without abstract base",
}
)
# UI/component importing infrastructure directly
if (
"ui component" in src_p
and "infrastructure" in tgt_p
and not edge.get("via_base")
):
edges.append(
{
"source": src,
"target": tgt,
"type": "VIOLATION",
"rule": "boundary",
"detail": "UI imports infrastructure without abstract base",
}
)
return nodes, edges
def _find_local_references(entities: list[dict], name: str) -> list[str]:
"""
Heuristic: check which local entities might reference the imported name.
If a class extends/implements the name, or has it in bases, it references it.
"""
refs = []
for ent in entities:
# Class extends/implements
if name in ent.get("bases", []):
refs.append(ent["name"])
# Decorator usage
if name in ent.get("decorators", []):
refs.append(ent["name"])
return refs
# ---------------------------------------------------------------------------
# LSP Client -- JSON-RPC over stdio
# ---------------------------------------------------------------------------
class LspClient:
"""
Minimal Language Server Protocol client using JSON-RPC over stdio.
Starts an LSP server as a subprocess, communicates via Content-Length
framed JSON-RPC messages, and dispatches responses to waiting callers
via a background reader thread.
Supports:
- initialize / initialized / shutdown / exit lifecycle
- textDocument/didOpen, textDocument/didClose
- textDocument/prepareCallHierarchy
- callHierarchy/outgoingCalls, callHierarchy/incomingCalls
- textDocument/references
"""
def __init__(self, cmd: list[str], cwd: str):
self._proc = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd,
)
assert self._proc.stdin is not None
assert self._proc.stdout is not None
assert self._proc.stderr is not None
self._stdin = self._proc.stdin
self._stdout = self._proc.stdout
self._stderr = self._proc.stderr
self._msg_id = 0
self._lock = threading.Lock()
self._events: dict[int, threading.Event] = {}
self._results: dict[int, Any] = {}
self._errors: dict[int, Any] = {}
self._alive = True
self._reader = threading.Thread(target=self._read_loop, daemon=True)
self._reader.start()
self._stderr_reader = threading.Thread(target=self._drain_stderr, daemon=True)
self._stderr_reader.start()
# -- Transport layer --
def _next_id(self) -> int:
with self._lock:
self._msg_id += 1
return self._msg_id
def _send(self, msg: dict) -> None:
"""Send a JSON-RPC message with Content-Length header."""
body = json.dumps(msg).encode("utf-8")
header = f"Content-Length: {len(body)}\r\n\r\n".encode("ascii")
try:
self._stdin.write(header + body)
self._stdin.flush()
except (BrokenPipeError, OSError):
self._alive = False
def _read_loop(self) -> None:
"""Background thread: read Content-Length framed messages from stdout."""
# Use os.read() for non-blocking reads. Python's BufferedReader.read(n)
# blocks until exactly n bytes are available, which deadlocks on pipes.
fd = self._stdout.fileno()
buf = b""
while self._alive:
try:
# Read until we have the full header separator
while b"\r\n\r\n" not in buf:
chunk = os.read(fd, 65536)
if not chunk:
self._alive = False
self._wake_all()
return
buf += chunk
header_end = buf.index(b"\r\n\r\n")
headers = buf[:header_end].decode("ascii")
buf = buf[header_end + 4 :]
content_length = 0
for line in headers.split("\r\n"):
if line.lower().startswith("content-length:"):
content_length = int(line.split(":", 1)[1].strip())
# Read body
while len(buf) < content_length:
chunk = os.read(fd, 65536)
if not chunk:
self._alive = False
self._wake_all()
return
buf += chunk
body = buf[:content_length]
buf = buf[content_length:]
try:
msg = json.loads(body.decode("utf-8"))
except json.JSONDecodeError:
continue
# Dispatch response
msg_id = msg.get("id")
if msg_id is not None and msg_id in self._events:
if "result" in msg:
self._results[msg_id] = msg["result"]
elif "error" in msg:
self._errors[msg_id] = msg["error"]
else:
# Null result is valid (e.g., empty references)
self._results[msg_id] = None
self._events[msg_id].set()
# Server-initiated notifications (diagnostics, progress) are ignored.
except Exception:
if self._alive:
continue
return
def _drain_stderr(self) -> None:
"""Drain stderr to prevent buffer deadlock."""
fd = self._stderr.fileno()
try:
while self._alive:
data = os.read(fd, 65536)
if not data:
return
except Exception:
pass
def _wake_all(self) -> None:
"""Wake all pending requests (server died or shutting down)."""
for event in self._events.values():
event.set()
# -- Request / Notification helpers --
def _request(
self, method: str, params: Any, timeout: float = LSP_REQUEST_TIMEOUT
) -> Any:
"""Send a request and wait for the response (blocking)."""
if not self._alive:
return None
msg_id = self._next_id()
event = threading.Event()
self._events[msg_id] = event
self._send({"jsonrpc": "2.0", "id": msg_id, "method": method, "params": params})
if event.wait(timeout):
self._events.pop(msg_id, None)
if msg_id in self._errors:
self._errors.pop(msg_id)
return None
return self._results.pop(msg_id, None)
# Timeout
self._events.pop(msg_id, None)
return None
def _request_fire(self, method: str, params: Any) -> int:
"""Send a request without waiting. Returns the message ID for later collection."""
if not self._alive:
return -1
msg_id = self._next_id()
event = threading.Event()
self._events[msg_id] = event
self._send({"jsonrpc": "2.0", "id": msg_id, "method": method, "params": params})
return msg_id
def _request_collect(
self, msg_id: int, timeout: float = LSP_REQUEST_TIMEOUT
) -> Any:
"""Wait for a previously fired request's response."""
if msg_id < 0 or msg_id not in self._events:
return None
event = self._events[msg_id]
if event.wait(timeout):
self._events.pop(msg_id, None)
if msg_id in self._errors:
self._errors.pop(msg_id)
return None
return self._results.pop(msg_id, None)
self._events.pop(msg_id, None)
return None
def _notify(self, method: str, params: Any) -> None:
"""Send a notification (no response expected)."""
self._send({"jsonrpc": "2.0", "method": method, "params": params})
# -- LSP lifecycle --
def initialize(self, root_uri: str) -> bool:
"""Send initialize request. Returns True on success."""
result = self._request(
"initialize",
{
"processId": os.getpid(),
"rootUri": root_uri,
"capabilities": {
"textDocument": {
"callHierarchy": {
"dynamicRegistration": False,
},
"references": {
"dynamicRegistration": False,
},
"synchronization": {
"didOpen": True,
"didClose": True,
},
},
},
"workspaceFolders": [
{"uri": root_uri, "name": os.path.basename(uri_to_path(root_uri))}
],
},
timeout=LSP_INIT_TIMEOUT,
)
if result is not None:
self._notify("initialized", {})
return True
return False
def shutdown(self) -> None:
"""Gracefully shut down the LSP server."""
if self._alive:
self._request("shutdown", None, timeout=5.0)
self._notify("exit", None)
self._alive = False
try:
self._proc.terminate()
self._proc.wait(timeout=5)
except Exception:
try:
self._proc.kill()
except Exception:
pass
@property
def alive(self) -> bool:
return self._alive and self._proc.poll() is None
# -- Document operations --
def did_open(self, uri: str, language_id: str, text: str) -> None:
"""Notify server that a document was opened."""
self._notify(
"textDocument/didOpen",
{
"textDocument": {
"uri": uri,
"languageId": language_id,
"version": 1,
"text": text,
},
},
)
def did_close(self, uri: str) -> None:
"""Notify server that a document was closed."""
self._notify(
"textDocument/didClose",
{
"textDocument": {"uri": uri},
},
)
# -- Query operations --
def document_symbols(
self,
uri: str,
timeout: float = LSP_REQUEST_TIMEOUT,
) -> list[dict]:
"""
Get all symbols in a document. Returns a tree of DocumentSymbol objects.
Each symbol has: name, kind (int), range, selectionRange, children.
SymbolKind: 5=Class, 6=Method, 11=Interface, 12=Function, 13=Variable,
10=Enum, 26=TypeParameter, 15=Namespace.
"""
result = self._request(
"textDocument/documentSymbol",
{
"textDocument": {"uri": uri},
},
timeout=timeout,
)
return result if isinstance(result, list) else []
def prepare_call_hierarchy(
self,
uri: str,
line: int,
character: int,
timeout: float = LSP_REQUEST_TIMEOUT,
) -> list[dict]:
"""
Prepare call hierarchy at position. Returns list of CallHierarchyItem.
Line and character are 0-based (LSP convention).
"""
result = self._request(
"textDocument/prepareCallHierarchy",
{
"textDocument": {"uri": uri},
"position": {"line": line, "character": character},
},
timeout=timeout,
)
return result if isinstance(result, list) else []
def outgoing_calls(
self,
item: dict,
timeout: float = LSP_REQUEST_TIMEOUT,
) -> list[dict]:
"""Get outgoing calls from a CallHierarchyItem."""
result = self._request(
"callHierarchy/outgoingCalls",
{
"item": item,
},
timeout=timeout,
)
return result if isinstance(result, list) else []
def incoming_calls(
self,
item: dict,
timeout: float = LSP_REQUEST_TIMEOUT,
) -> list[dict]:
"""Get incoming calls to a CallHierarchyItem."""
result = self._request(
"callHierarchy/incomingCalls",
{
"item": item,
},
timeout=timeout,
)
return result if isinstance(result, list) else []
def references(
self,
uri: str,
line: int,
character: int,
timeout: float = LSP_REQUEST_TIMEOUT,
) -> list[dict]:
"""
Find all references to the symbol at position.
Returns list of Location {uri, range}.
"""
result = self._request(
"textDocument/references",
{
"textDocument": {"uri": uri},
"position": {"line": line, "character": character},
"context": {"includeDeclaration": False},
},
timeout=timeout,
)
return result if isinstance(result, list) else []
# ---------------------------------------------------------------------------
# Project discovery for LSP
# ---------------------------------------------------------------------------
def discover_projects(root: str, file_results: list[FileResult]) -> list[ProjectGroup]:
"""
Group files by LSP project root, detected by presence of config files.
For TypeScript: looks for tsconfig.json.
For Python: looks for pyrightconfig.json or pyproject.toml.
For PHP: looks for composer.json.
Files that do not belong to any detected project are skipped.
"""
# Find all config files
ts_configs: list[str] = []
py_configs: list[str] = []
php_configs: list[str] = []
for dirpath, dirnames, filenames in os.walk(root):
# Skip unwanted directories
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
for fname in filenames:
full = os.path.join(dirpath, fname)
if fname == "tsconfig.json":
ts_configs.append(full)
elif fname in ("pyrightconfig.json", "pyproject.toml"):
py_configs.append(full)
elif fname == "composer.json":
php_configs.append(full)
# Sort configs by depth (deepest first) so files match the most specific project
ts_configs.sort(key=lambda p: -p.count(os.sep))
py_configs.sort(key=lambda p: -p.count(os.sep))
php_configs.sort(key=lambda p: -p.count(os.sep))
projects: list[ProjectGroup] = []
assigned_files: set[str] = set()
# Assign TypeScript files to projects
for config in ts_configs:
proj_root = os.path.dirname(config)
proj = ProjectGroup(
root=proj_root,
language="typescript",
config_file=config,
)
for r in file_results:
if r.language == "typescript" and r.path not in assigned_files:
if r.path.startswith(proj_root + os.sep):
proj.files.append(r)
assigned_files.add(r.path)
if proj.files:
projects.append(proj)
# Assign Python files to projects
for config in py_configs:
proj_root = os.path.dirname(config)
proj = ProjectGroup(
root=proj_root,
language="python",
config_file=config,
)
for r in file_results:
if r.language == "python" and r.path not in assigned_files:
if r.path.startswith(proj_root + os.sep):
proj.files.append(r)
assigned_files.add(r.path)
if proj.files:
projects.append(proj)
# Assign PHP files to projects
for config in php_configs:
proj_root = os.path.dirname(config)
proj = ProjectGroup(
root=proj_root,
language="php",
config_file=config,
)
for r in file_results:
if r.language == "php" and r.path not in assigned_files:
if r.path.startswith(proj_root + os.sep):
proj.files.append(r)
assigned_files.add(r.path)
if proj.files:
projects.append(proj)
return projects
# ---------------------------------------------------------------------------
# LSP analysis -- call hierarchy + references
# ---------------------------------------------------------------------------
def _language_id(language: str, path: str) -> str:
"""Map internal language + file extension to LSP languageId."""
if language == "python":
return "python"
if language == "php":
return "php"
if path.endswith(".tsx"):
return "typescriptreact"
return "typescript"
# LSP SymbolKind constants
_SK_CLASS = 5
_SK_METHOD = 6
_SK_PROPERTY = 7
_SK_ENUM = 10
_SK_INTERFACE = 11
_SK_FUNCTION = 12
_SK_VARIABLE = 13
_SK_CONSTANT = 14
_SK_ENUM_MEMBER = 22
_SK_TYPE_PARAMETER = 26
_SK_TO_KIND = {
_SK_CLASS: "class",
_SK_METHOD: "method",
_SK_INTERFACE: "interface",
_SK_FUNCTION: "function",
_SK_VARIABLE: "function", # const exports
_SK_CONSTANT: "function",
_SK_ENUM: "enum",
_SK_TYPE_PARAMETER: "type",
_SK_PROPERTY: "function",
}
def _sym_line(sym: dict) -> int:
"""Extract 1-based line from a DocumentSymbol or SymbolInformation."""
# DocumentSymbol format: { selectionRange: { start: { line } } }
sel = sym.get("selectionRange")
if sel:
return sel.get("start", {}).get("line", 0) + 1
# SymbolInformation format: { location: { range: { start: { line } } } }
loc = sym.get("location")
if loc:
return loc.get("range", {}).get("start", {}).get("line", 0) + 1
# Fallback to range
rng = sym.get("range")
if rng:
return rng.get("start", {}).get("line", 0) + 1
return 0
def _lsp_symbols_to_entities(symbols: list[dict]) -> tuple[list[dict], list[str]]:
"""
Convert LSP symbol results into Entity dicts and exported names.
Handles both DocumentSymbol (nested, has children) and SymbolInformation
(flat, has location) formats. Filters to top-level definitions only:
classes, interfaces, functions, types, enums, and exported constants.
"""
entities: list[dict] = []
exported_names: list[str] = []
seen_names: set[str] = set()
# Detect format: DocumentSymbol has "children" or "selectionRange",
# SymbolInformation has "location"
is_doc_symbol = any("selectionRange" in s or "children" in s for s in symbols[:5])
if is_doc_symbol:
# DocumentSymbol: only process top-level symbols
for sym in symbols:
_process_doc_symbol(sym, entities, exported_names, seen_names)
else:
# SymbolInformation: flat list, filter to top-level kinds
for sym in symbols:
# Skip methods/properties (they belong to a class, not top-level)
if sym.get("containerName", ""):
continue
_process_symbol_info(sym, entities, exported_names, seen_names)
return entities, exported_names
def _process_doc_symbol(
sym: dict,
entities: list[dict],
exported_names: list[str],
seen_names: set[str],
) -> None:
"""Process a single DocumentSymbol (with children)."""
kind_int = sym.get("kind", 0)
name = sym.get("name", "")
line = _sym_line(sym)
if not name or name in seen_names:
return
children = sym.get("children", [])
if kind_int in (_SK_CLASS, _SK_INTERFACE):
methods = []
for child in children:
child_kind = child.get("kind", 0)
child_name = child.get("name", "")
if child_kind in (_SK_METHOD, _SK_PROPERTY, _SK_FUNCTION) and child_name:
methods.append(child_name)
kind = "class" if kind_int == _SK_CLASS else "interface"
entities.append(
{
"name": name,
"kind": kind,
"line": line,
"methods": methods,
"bases": [],
"decorators": [],
}
)
seen_names.add(name)
exported_names.append(name)
elif kind_int == _SK_ENUM:
members = [
c.get("name", "") for c in children if c.get("kind") == _SK_ENUM_MEMBER
]
entities.append(
{
"name": name,
"kind": "enum",
"line": line,
"methods": members,
"bases": [],
"decorators": [],
}
)
seen_names.add(name)
exported_names.append(name)
elif kind_int == _SK_FUNCTION:
entities.append(
{
"name": name,
"kind": "function",
"line": line,
"methods": [],
"bases": [],
"decorators": [],
}
)
seen_names.add(name)
exported_names.append(name)
elif kind_int in (_SK_VARIABLE, _SK_CONSTANT):
# Only capture exported-looking names (PascalCase or camelCase identifiers)
# Skip destructured locals, parameters, etc.
if len(name) >= 2 and (name[0].isupper() or not name.startswith("_")):
ent_kind = "type" if name[0].isupper() and not children else "function"
methods = []
if children:
for child in children:
child_name = child.get("name", "")
if child_name:
methods.append(child_name)
entities.append(
{
"name": name,
"kind": ent_kind,
"line": line,
"methods": methods,
"bases": [],
"decorators": [],
}
)
seen_names.add(name)
exported_names.append(name)
elif kind_int == _SK_TYPE_PARAMETER:
entities.append(
{
"name": name,
"kind": "type",
"line": line,
"methods": [],
"bases": [],
"decorators": [],
}
)
seen_names.add(name)
exported_names.append(name)
def _process_symbol_info(
sym: dict,
entities: list[dict],
exported_names: list[str],
seen_names: set[str],
) -> None:
"""Process a single SymbolInformation (flat, no children)."""
kind_int = sym.get("kind", 0)
name = sym.get("name", "")
line = _sym_line(sym)
if not name or name in seen_names:
return
kind = _SK_TO_KIND.get(kind_int)
if not kind:
return
entities.append(
{
"name": name,
"kind": kind,
"line": line,
"methods": [],
"bases": [],
"decorators": [],
}
)
seen_names.add(name)
exported_names.append(name)
def _drain_prepare_batch(
client: "LspClient",
batch: list[tuple[int, str, dict]],
path_to_node_id: dict[str, str],
source_node_id: str,
lsp_edges: list[LspEdge],
timeout: float,
) -> int:
"""
Collect prepareCallHierarchy results, pipeline outgoingCalls, collect those.
Returns number of CALLS edges found.
"""
calls_found = 0
# Collect prepare results and fire outgoingCalls
outgoing_pending: list[tuple[int, str, str]] = [] # (msg_id, ent_name, node_id)
for mid, ent_name, _ent in batch:
items = client._request_collect(mid, timeout=timeout)
if not isinstance(items, list):
continue
for item in items:
if item.get("name") != ent_name:
continue
oc_mid = client._request_fire("callHierarchy/outgoingCalls", {"item": item})
if oc_mid >= 0:
outgoing_pending.append((oc_mid, ent_name, source_node_id))
# Collect outgoingCalls results
for oc_mid, ent_name, node_id in outgoing_pending:
outgoing = client._request_collect(oc_mid, timeout=timeout)
if not isinstance(outgoing, list):
continue
for call in outgoing:
target = call.get("to", {})
target_uri = target.get("uri", "")
target_name = target.get("name", "")
target_path = uri_to_path(target_uri)
target_node_id = path_to_node_id.get(target_path)
if not target_node_id:
continue
from_ranges = [fr["start"]["line"] + 1 for fr in call.get("fromRanges", [])]
lsp_edges.append(
LspEdge(
source=f"{node_id}::{ent_name}",
target=f"{target_node_id}::{target_name}",
edge_type="CALLS",
metadata={"from_lines": from_ranges, "lsp_verified": True},
)
)
calls_found += 1
return calls_found
def _find_enclosing_entity(
entities: list[dict],
line_0based: int,
) -> Optional[str]:
"""
Find the entity that encloses the given 0-based line.
Uses a simple heuristic: the entity whose line is closest to (but not
after) the target line.
"""
best: Optional[dict] = None
for ent in entities:
ent_line_0 = ent["line"] - 1 # entity lines are 1-based
if ent_line_0 <= line_0based:
if best is None or ent_line_0 > best["line"] - 1:
best = ent
return best["name"] if best else None
def analyze_project_lsp(
project: ProjectGroup,
root: str,
path_to_node_id: dict[str, str],
node_id_to_result: dict[str, FileResult],
request_timeout: float,
) -> list[LspEdge]:
"""
Analyze a single project with LSP for precise call graph and references.
Returns a list of LspEdge instances to merge into the main graph.
"""
# Determine LSP server command
if project.language == "typescript":
cmd = ["typescript-language-server", "--stdio"]
elif project.language == "php":
# phpactor language-server: JSON-RPC over stdio.
# Install: composer global require phpactor/phpactor
cmd = ["phpactor", "language-server"]
else:
cmd = ["pyright-langserver", "--stdio"]
# Check server availability
server_bin = cmd[0]
from shutil import which
if not which(server_bin):
print(f" [lsp] WARNING: {server_bin} not found, skipping {project.root}")
return []
proj_name = os.path.relpath(project.root, root)
print(f" [lsp] Starting {server_bin} for {proj_name} ({len(project.files)} files)")
client = LspClient(cmd, cwd=project.root)
lsp_edges: list[LspEdge] = []
try:
# Initialize
root_uri = path_to_uri(project.root)
if not client.initialize(root_uri):
print(f" [lsp] WARNING: initialize failed for {proj_name}")
return []
# Open all project files (batch)
file_contents: dict[str, str] = {}
for r in project.files:
try:
with open(r.path, "r", encoding="utf-8", errors="replace") as f:
content = f.read()
file_contents[r.path] = content
uri = path_to_uri(r.path)
lang_id = _language_id(r.language, r.path)
client.did_open(uri, lang_id, content)
except OSError:
continue
# Give the server a moment to process opened files
time.sleep(1.0)
# -- Phase 1: Merge LSP documentSymbol with regex entities --
# LSP gives precise line numbers and finds entities regex misses
# (object literals, Zod schemas). Regex gives method lists via
# brace scanning. Merge: take LSP entities as base, carry over
# regex methods for classes/interfaces that LSP returns without children.
symbols_replaced = 0
for r in project.files:
if not client.alive:
break
node_id = path_to_node_id.get(r.path)
if not node_id:
continue
uri = path_to_uri(r.path)
lsp_syms = client.document_symbols(uri, timeout=request_timeout)
if lsp_syms:
new_entities, new_exports = _lsp_symbols_to_entities(lsp_syms)
# Build regex method index: entity_name -> methods list
regex_methods: dict[str, list[str]] = {}
for old_ent in r.entities:
if old_ent.get("methods"):
regex_methods[old_ent["name"]] = old_ent["methods"]
# Merge: carry over regex methods when LSP has none
for ent in new_entities:
if ent["kind"] in ("class", "interface") and not ent.get("methods"):
regex_m = regex_methods.get(ent["name"])
if regex_m:
ent["methods"] = regex_m
# Also carry over bases/decorators from regex
if not ent.get("bases") or not ent.get("decorators"):
for old_ent in r.entities:
if old_ent["name"] == ent["name"]:
if not ent.get("bases") and old_ent.get("bases"):
ent["bases"] = old_ent["bases"]
if not ent.get("decorators") and old_ent.get(
"decorators"
):
ent["decorators"] = old_ent["decorators"]
break
# Add regex entities not found by LSP (e.g., Zod schemas)
lsp_names = {e["name"] for e in new_entities}
for old_ent in r.entities:
if old_ent["name"] not in lsp_names:
new_entities.append(old_ent)
if old_ent["name"] not in new_exports:
new_exports.append(old_ent["name"])
r.entities = new_entities
r.exported_names = list(set(new_exports))
symbols_replaced += 1
if symbols_replaced:
print(
f" [lsp] {proj_name}: merged entities in {symbols_replaced} files via documentSymbol"
)
# -- Phase 2: Pipelined call hierarchy + references --
# Strategy: for each file, fire all prepareCallHierarchy and
# references requests at once (pipelined), then collect results.
# This overlaps network round-trips with server processing.
total_entities = sum(len(r.entities) for r in project.files)
processed = 0
calls_found = 0
refs_found = 0
PIPELINE_BATCH = 40 # max concurrent in-flight requests
for r in project.files:
if not client.alive:
print(f" [lsp] WARNING: server died while processing {proj_name}")
break
content = file_contents.get(r.path)
if not content:
continue
node_id = path_to_node_id.get(r.path)
if not node_id:
continue
uri = path_to_uri(r.path)
# -- Batch 1: fire prepareCallHierarchy + references for all entities --
pending_prepare: list[tuple[int, str, dict]] = [] # (msg_id, ent_name, ent)
pending_refs: list[tuple[int, str, str]] = [] # (msg_id, ent_name, node_id)
for ent in r.entities:
ent_name = ent["name"]
ent_line_0 = ent["line"] - 1
ent_col = find_entity_column(content, ent_name, ent["line"])
if ent["kind"] in ("function", "class"):
mid = client._request_fire(
"textDocument/prepareCallHierarchy",
{
"textDocument": {"uri": uri},
"position": {"line": ent_line_0, "character": ent_col},
},
)
if mid >= 0:
pending_prepare.append((mid, ent_name, ent))
if ent_name in r.exported_names:
mid = client._request_fire(
"textDocument/references",
{
"textDocument": {"uri": uri},
"position": {"line": ent_line_0, "character": ent_col},
"context": {"includeDeclaration": False},
},
)
if mid >= 0:
pending_refs.append((mid, ent_name, node_id))
# Drain pipeline if too many in-flight
while len(pending_prepare) + len(pending_refs) >= PIPELINE_BATCH:
calls_found += _drain_prepare_batch(
client,
pending_prepare[: PIPELINE_BATCH // 2],
path_to_node_id,
node_id,
lsp_edges,
request_timeout,
)
pending_prepare = pending_prepare[PIPELINE_BATCH // 2 :]
# -- Collect remaining prepareCallHierarchy results --
calls_found += _drain_prepare_batch(
client,
pending_prepare,
path_to_node_id,
node_id,
lsp_edges,
request_timeout,
)
# -- Collect references results --
for mid, ent_name, src_node_id in pending_refs:
ref_locations = client._request_collect(mid, timeout=request_timeout)
if not isinstance(ref_locations, list):
continue
for loc in ref_locations:
ref_uri = loc.get("uri", "")
ref_path = uri_to_path(ref_uri)
ref_node_id = path_to_node_id.get(ref_path)
if not ref_node_id or ref_node_id == src_node_id:
continue
ref_result = node_id_to_result.get(ref_node_id)
ref_line_0 = loc.get("range", {}).get("start", {}).get("line", 0)
ref_entity_name = None
if ref_result:
ref_entity_name = _find_enclosing_entity(
ref_result.entities,
ref_line_0,
)
source_key = (
f"{ref_node_id}::{ref_entity_name}"
if ref_entity_name
else ref_node_id
)
target_key = f"{src_node_id}::{ent_name}"
lsp_edges.append(
LspEdge(
source=source_key,
target=target_key,
edge_type="REFERENCES",
metadata={"lsp_verified": True, "ref_line": ref_line_0 + 1},
)
)
refs_found += 1
processed += len(r.entities)
if processed % 200 < len(r.entities):
print(
f" [lsp] {proj_name}: {processed}/{total_entities} entities "
f"({calls_found} calls, {refs_found} refs)"
)
client.did_close(uri)
print(
f" [lsp] {proj_name}: done -- {calls_found} calls, {refs_found} refs from {processed} entities"
)
except Exception as exc:
print(f" [lsp] ERROR in {proj_name}: {exc}")
finally:
client.shutdown()
return lsp_edges
# ---------------------------------------------------------------------------
# LSP merge -- integrate LSP edges into the graph
# ---------------------------------------------------------------------------
def merge_lsp_edges(
edges: list[dict],
lsp_edges: list[LspEdge],
) -> list[dict]:
"""
Merge LSP-derived edges into the existing edge list.
- New CALLS edges are added directly.
- REFERENCES edges that match existing ones get ``lsp_verified: true``.
- New REFERENCES edges not found by the regex pass are added.
"""
# Index existing edges for fast lookup
existing_keys: dict[tuple, int] = {}
for idx, edge in enumerate(edges):
key = (edge["source"], edge["target"], edge["type"])
existing_keys[key] = idx
new_edges: list[dict] = []
verified_count = 0
new_count = 0
for le in lsp_edges:
key = (le.source, le.target, le.edge_type)
if key in existing_keys:
# Mark existing edge as LSP-verified
edges[existing_keys[key]]["lsp_verified"] = True
verified_count += 1
else:
# Add new edge
edge_dict: dict = {
"source": le.source,
"target": le.target,
"type": le.edge_type,
}
edge_dict.update(le.metadata)
new_edges.append(edge_dict)
new_count += 1
# Track so we don't add duplicates
existing_keys[key] = -1
print(f"[belief-map] LSP merge: {verified_count} verified, {new_count} new edges")
return edges + new_edges
# ---------------------------------------------------------------------------
# LSP enrichment orchestrator
# ---------------------------------------------------------------------------
def enrich_with_lsp(
root: str,
results: list[FileResult],
nodes: list[dict],
edges: list[dict],
request_timeout: float,
) -> tuple[list[dict], list[dict]]:
"""
Run LSP analysis on all detected projects and merge edges into the graph.
For TypeScript: replaces regex-extracted entities with LSP documentSymbol
results, then queries call hierarchy and references for precise edges.
For Python: queries call hierarchy and references only (keeps AST entities).
"""
# Build lookup tables
path_to_node_id: dict[str, str] = {}
node_id_to_result: dict[str, FileResult] = {}
for r in results:
nid = make_node_id(r.path, root)
path_to_node_id[r.path] = nid
node_id_to_result[nid] = r
# Discover projects
projects = discover_projects(root, results)
if not projects:
print(
"[belief-map] No LSP-compatible projects found "
"(no tsconfig.json, pyrightconfig.json, or composer.json)"
)
return nodes, edges
ts_projects = [p for p in projects if p.language == "typescript"]
py_projects = [p for p in projects if p.language == "python"]
php_projects = [p for p in projects if p.language == "php"]
ts_files = sum(len(p.files) for p in ts_projects)
py_files = sum(len(p.files) for p in py_projects)
php_files = sum(len(p.files) for p in php_projects)
print(
f"[belief-map] LSP: {len(projects)} projects "
f"({len(ts_projects)} TS/{ts_files} files, "
f"{len(py_projects)} Py/{py_files} files, "
f"{len(php_projects)} PHP/{php_files} files)"
)
all_lsp_edges: list[LspEdge] = []
for project in projects:
proj_edges = analyze_project_lsp(
project,
root,
path_to_node_id,
node_id_to_result,
request_timeout,
)
all_lsp_edges.extend(proj_edges)
if all_lsp_edges:
edges = merge_lsp_edges(edges, all_lsp_edges)
# Rebuild nodes with LSP-replaced entities
node_by_id = {n["id"]: n for n in nodes}
for r in results:
nid = path_to_node_id.get(r.path)
if nid and nid in node_by_id:
node_by_id[nid]["entities"] = r.entities
nodes = list(node_by_id.values())
return nodes, edges
# ---------------------------------------------------------------------------
# Output formatters
# ---------------------------------------------------------------------------
def _sexp_escape(s: str) -> str:
"""Escape a string for S-expression output (double-quote delimited)."""
return s.replace("\\", "\\\\").replace('"', '\\"')
def _entity_id(module_path: str, entity_name: str) -> str:
"""Generate a short unique ID for an entity based on path + name SHA."""
raw = f"{module_path}::{entity_name}"
return hashlib.sha1(raw.encode()).hexdigest()[:8]
# ---------------------------------------------------------------------------
# Path trie for compressed path definitions
# ---------------------------------------------------------------------------
def _build_path_tree(path_sid_pairs: list[tuple[str, str]]) -> list[str]:
"""
Build a compressed path trie and emit as indented S-expression tree.
Shares common path prefixes to reduce repetition. Collapses single-child
chains into combined segments (path compression).
Output format::
(paths
(agent-hub/packages/agent-research/src
(ah0 ReasoningEngine)
(ah1 ResearchAgent))
(app-service/app-service/src/operatives
(asp340 operatives.controller)))
"""
# Build trie: dict of dicts, leaves are {"_sid": "ah0"}
# A node can be BOTH a directory (has children) and a leaf (has _sid)
# when index.ts strips to the directory name.
root: dict = {}
for full_path, sid in path_sid_pairs:
segments = full_path.split("/")
node = root
for seg in segments[:-1]:
node = node.setdefault(seg, {})
# Leaf: store sid. If the key already exists as a dict (directory),
# merge _sid into it rather than overwriting children.
leaf_key = segments[-1]
existing = node.get(leaf_key)
if isinstance(existing, dict) and "_sid" not in existing:
existing["_sid"] = sid
else:
node[leaf_key] = {"_sid": sid}
# Compress: collapse single-child internal nodes
def compress(node: dict) -> dict:
result: dict = {}
for key, child in node.items():
if isinstance(child, dict) and "_sid" in child:
result[key] = child # leaf, keep as-is
elif isinstance(child, dict):
compressed = compress(child)
# If single non-leaf child, merge keys
while len(compressed) == 1:
only_key, only_val = next(iter(compressed.items()))
if isinstance(only_val, dict) and "_sid" not in only_val:
key = f"{key}/{only_key}"
compressed = only_val
else:
break
result[key] = compressed
else:
result[key] = child
return result
compressed = compress(root)
# Emit as indented S-expressions
def emit(node: dict, depth: int) -> list[str]:
lines: list[str] = []
indent = " " * depth
for key in sorted(node.keys()):
if key == "_sid":
continue # handled by parent
child = node[key]
if isinstance(child, dict) and "_sid" in child:
# Check if this is BOTH a leaf and a directory (has other keys)
other_keys = [k for k in child if k != "_sid"]
if other_keys:
# Emit the _sid as a :self entry, then recurse for children
lines.append(f"{indent}({key}")
lines.append(f"{indent} ({child['_sid']} :self)")
lines.extend(emit(child, depth + 1))
lines.append(f"{indent})")
else:
lines.append(f"{indent}({child['_sid']} {key})")
elif isinstance(child, dict):
lines.append(f"{indent}({key}")
lines.extend(emit(child, depth + 1))
lines.append(f"{indent})")
return lines
return emit(compressed, 1)
def write_sexp(
output_path: str,
nodes: list[dict],
edges: list[dict],
_root: str,
mode: str,
total_files: int,
) -> None:
"""
Write the belief map as flat S-expression facts with path-map compression.
The file starts with ``(def <id> <full-path>)`` bindings that assign short
numeric IDs to every module path. All subsequent lines use these IDs instead
of full paths, reducing file size by ~60-70%.
Format::
; --- paths ---
(def 0 agent-hub/packages/agent-research/src/ReasoningEngine)
(def 1 agent-hub/packages/agent-research/src/ResearchAgent)
; --- nodes ---
(node 0 ty "general module" :naming PascalCase :pkg agent-hub)
; --- entities ---
(fn 0 isToolIncluded 5)
(cls 1 ResearchAgent 322 (:bases BaseAgent) (:methods ...))
; --- edges ---
(imports 1 0 :via-base)
(refs 1 0::ReasoningSchemaType)
; --- violations ---
(violation boundary 42 99 "domain imports infrastructure without abstract base")
"""
sorted_nodes = sorted(nodes, key=lambda n: n["id"])
# Build path map: full path -> short prefixed ID (repo-prefix + seq number)
# e.g. ah0 = agent-hub first module, dr42 = drive 43rd module
_REPO_PREFIX: dict[str, str] = {}
_repo_counters: dict[str, int] = {}
path_to_sid: dict[str, str] = {}
for node in sorted_nodes:
repo = node.get("repo", "unknown")
if repo not in _REPO_PREFIX:
# Build 2-char prefix from repo name
parts = repo.replace("-", " ").split()
if len(parts) >= 2:
prefix = parts[0][0] + parts[1][0]
else:
prefix = repo[:2]
# Deduplicate prefixes
base = prefix
idx = 2
while prefix in _REPO_PREFIX.values():
prefix = (
base + repo[idx : idx + 1] if idx < len(repo) else base + str(idx)
)
idx += 1
_REPO_PREFIX[repo] = prefix
_repo_counters[repo] = 0
seq = _repo_counters[repo]
_repo_counters[repo] = seq + 1
sid = f"{_REPO_PREFIX[repo]}{seq}"
path_to_sid[node["id"]] = sid
def _pid(path: str) -> str:
"""Resolve a path (possibly with ::entity suffix) to short ID."""
if "::" in path:
mod, ent = path.split("::", 1)
sid = path_to_sid.get(mod)
return f"{sid}::{ent}" if sid is not None else path
sid = path_to_sid.get(path)
return sid if sid is not None else path
lines: list[str] = []
violations = [e for e in edges if e.get("type") == "VIOLATION"]
real_edges = [e for e in edges if e.get("type") != "VIOLATION"]
lines.append(f"; Belief Map")
lines.append(f"; generated {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}")
lines.append(f"; mode {mode}")
lines.append(
f"; {total_files} files {len(sorted_nodes)} nodes {len(real_edges)} edges {len(violations)} violations"
)
lines.append("")
# -- Path tree (compressed trie) --
lines.append("; --- paths ---")
trie_pairs = [(node["id"], path_to_sid[node["id"]]) for node in sorted_nodes]
lines.append("(paths")
lines.extend(_build_path_tree(trie_pairs))
lines.append(")")
lines.append("")
# -- Nodes with invariant (Gap 1 fix) --
lines.append("; --- nodes ---")
for node in sorted_nodes:
sid = path_to_sid[node["id"]]
_LANG_SHORT = {
"python": "py",
"typescript": "ts",
"tsx": "tsx",
"jsx": "jsx",
"javascript": "js",
"php": "php",
}
lang = _LANG_SHORT.get(node["language"], node["language"][:2])
purpose = _sexp_escape(node.get("purpose", ""))
inv = node.get("invariant", {})
naming = inv.get("naming", "mixed")
pkg = inv.get("package", "")
lines.append(f'(node {sid} {lang} "{purpose}" :naming {naming} :pkg {pkg})')
lines.append("")
# -- Entities (each has a unique eid based on sha(path::name)) --
lines.append("; --- entities ---")
for node in sorted_nodes:
sid = path_to_sid[node["id"]]
for ent in node.get("entities", []):
kind = ent["kind"]
name = ent["name"]
line_num = ent.get("line", 0)
eid = _entity_id(node["id"], name)
kind_tag = {
"class": "cls",
"function": "fn",
"interface": "ifc",
"type": "typ",
"enum": "enm",
"trait": "trt",
}.get(kind, kind[:3])
bases = ent.get("bases", [])
deco = ent.get("decorators", [])
methods = ent.get("methods", [])
if not bases and not deco and not methods:
lines.append(f"({kind_tag} {sid} {name} {line_num} :eid {eid})")
else:
parts = [f"({kind_tag} {sid} {name} {line_num} :eid {eid}"]
if bases:
parts.append(f"(:bases {' '.join(bases)})")
if deco:
parts.append(f"(:deco {' '.join(deco)})")
if methods:
parts.append(f"(:methods {' '.join(methods)})")
lines.append(" ".join(parts) + ")")
lines.append("")
# -- Edges --
lines.append("; --- edges ---")
sorted_edges = sorted(
real_edges, key=lambda e: (e["type"], e["source"], e["target"])
)
for edge in sorted_edges:
src = _pid(edge["source"])
tgt = _pid(edge["target"])
etype = edge["type"]
if etype == "IMPORTS":
flags = " :via-base" if edge.get("via_base") else ""
lsp = " :lsp" if edge.get("lsp_verified") else ""
lines.append(f"(imports {src} {tgt}{flags}{lsp})")
elif etype == "CALLS_API":
lsp = " :lsp" if edge.get("lsp_verified") else ""
lines.append(f"(calls-api {src} {tgt} :via-ifc{lsp})")
elif etype == "DATA_FLOWS_TO":
flags = " :validated" if edge.get("validated") else ""
lines.append(f"(data-flow {src} {tgt}{flags})")
elif etype == "REFERENCES":
lsp = " :lsp" if edge.get("lsp_verified") else ""
lines.append(f"(refs {src} {tgt}{lsp})")
elif etype == "CALLS":
fl = edge.get("from_lines", [])
lns = f" :lines {' '.join(str(l) for l in fl)}" if fl else ""
lines.append(f"(calls {src} {tgt}{lns} :lsp)")
elif etype == "HTTP_CALLS":
client = edge.get("client_entity", "")
client_str = f" :client {client}" if client else ""
lines.append(f"(http-calls {src} {tgt} :transport http{client_str})")
# -- Violations (Gap 2 fix) --
if violations:
lines.append("")
lines.append("; --- violations ---")
for v in violations:
src = _pid(v["source"])
tgt = _pid(v["target"])
rule = v.get("rule", "unknown")
detail = _sexp_escape(v.get("detail", ""))
lines.append(f'(violation {rule} {src} {tgt} "{detail}")')
with open(output_path, "w") as f:
f.write("\n".join(lines))
f.write("\n")
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
args = sys.argv[1:]
full_rebuild = "--full" in args
use_lsp = "--lsp" in args
lsp_timeout = LSP_REQUEST_TIMEOUT
# Parse --lsp-timeout N
known_flags = {"--full", "--lsp"}
clean_args = []
i = 0
while i < len(args):
if args[i] == "--lsp-timeout" and i + 1 < len(args):
try:
lsp_timeout = float(args[i + 1])
except ValueError:
print(
f"Error: invalid --lsp-timeout value: {args[i + 1]}",
file=sys.stderr,
)
sys.exit(1)
i += 2
continue
if args[i] not in known_flags:
clean_args.append(args[i])
i += 1
root = os.path.abspath(clean_args[0] if clean_args else ".")
if not os.path.isdir(root):
print(f"Error: {root} is not a directory", file=sys.stderr)
sys.exit(1)
t0 = time.monotonic()
mode = "LSP-enhanced" if use_lsp else "regex/AST"
print(f"[belief-map] Scanning {root} ... (mode: {mode})")
files = discover_files(root)
py_count = sum(1 for _, l, _ in files if l == "python")
ts_count = sum(1 for _, l, _ in files if l == "typescript")
php_count = sum(1 for _, l, _ in files if l == "php")
print(
f"[belief-map] Found {len(files)} source files "
f"({py_count} py, {ts_count} ts, {php_count} php)"
)
cache = {} if full_rebuild else load_cache(root)
to_parse: list[tuple[str, str, str]] = []
cached_results: list[FileResult] = []
for path, lang, repo in files:
entry = cache.get(path)
if entry and not full_rebuild:
try:
cur_mtime = os.path.getmtime(path)
if cur_mtime <= entry.get("mtime", 0):
cached_results.append(FileResult(**entry["result"]))
continue
except OSError:
pass
to_parse.append((path, lang, repo))
print(
f"[belief-map] Parsing {len(to_parse)} changed files ({len(cached_results)} cached)"
)
new_results: list[FileResult] = []
if to_parse:
workers = min(os.cpu_count() or 4, len(to_parse))
with ProcessPoolExecutor(max_workers=workers) as pool:
futures = {pool.submit(parse_file, item): item for item in to_parse}
for fut in as_completed(futures):
result = fut.result()
if result:
new_results.append(result)
# Update cache
new_cache: dict = {}
all_results = cached_results + new_results
for r in all_results:
new_cache[r.path] = {
"mtime": r.mtime,
"result": {
"path": r.path,
"language": r.language,
"repo": r.repo,
"mtime": r.mtime,
"content_hash": r.content_hash,
"imports": r.imports,
"exports_abstract": r.exports_abstract,
"implements": r.implements,
"extends": r.extends,
"purpose": r.purpose,
"naming_convention": r.naming_convention,
"has_validation": r.has_validation,
"entities": r.entities,
"imported_names": r.imported_names,
"exported_names": r.exported_names,
"traits_used": r.traits_used,
},
}
save_cache(root, new_cache)
# Build graph (regex/AST pass)
nodes, edges = build_graph(all_results, root)
t_regex = time.monotonic() - t0
print(
f"[belief-map] Regex/AST pass: {t_regex:.2f}s -- {len(nodes)} nodes, {len(edges)} edges"
)
# LSP enrichment pass (if requested)
if use_lsp:
t_lsp_start = time.monotonic()
print(
f"[belief-map] Starting LSP enrichment (timeout={lsp_timeout}s per request) ..."
)
nodes, edges = enrich_with_lsp(root, all_results, nodes, edges, lsp_timeout)
t_lsp = time.monotonic() - t_lsp_start
print(f"[belief-map] LSP pass: {t_lsp:.2f}s")
# Write output
output_path = os.path.join(root, OUTPUT_FILE)
write_sexp(output_path, nodes, edges, root, mode, len(all_results))
elapsed = time.monotonic() - t0
print(
f"[belief-map] Done in {elapsed:.2f}s -- {len(nodes)} nodes, {len(edges)} edges"
)
print(f"[belief-map] Output: {output_path}")
# Stats
repos = sorted(set(n["repo"] for n in nodes))
print(f"[belief-map] Repos: {', '.join(repos)}")
from collections import Counter
edge_types = Counter(e["type"] for e in edges)
total_entities = sum(len(n.get("entities", [])) for n in nodes)
lsp_verified = sum(1 for e in edges if e.get("lsp_verified"))
print(f"[belief-map] Entities: {total_entities}")
for etype, count in sorted(edge_types.items()):
print(f"[belief-map] {etype}: {count}")
if lsp_verified:
print(f"[belief-map] LSP-verified: {lsp_verified}")
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink