This commit is contained in:
Dion Moult
2026-07-26 18:03:09 +10:00
parent 7aa967b01a
commit 291d7d8441
72 changed files with 477 additions and 377 deletions
@@ -23,6 +23,7 @@ import operator
import collections
import bootstrap
class Node:
def __init__(self, s, loc, tokens, rule=None):
self.rule = rule or (type(self).__name__)
@@ -58,15 +59,15 @@ class ListNode:
rules_as_list = set()
for t in self.tokens:
r = getattr(t, 'rule', None)
r = getattr(t, "rule", None)
if r:
rules_as_list.add(r)
self.dict_tokens[r].append(t)
for r, t in tokens.asDict().items():
if r not in rules_as_list:
self.dict_tokens[r].append(t)
self.flat = sum([getattr(t, "flat", [t]) for t in self.tokens], [])
def __repr__(self):
@@ -74,7 +75,7 @@ class ListNode:
def __iter__(self):
return iter(self.tokens)
# Somehow indexing messes up the pyparsing results, so instead of x[0] use list(x)[0]
# def __getitem__(self, i):
# return self.tokens[i]
@@ -110,7 +111,7 @@ def format_clause(exp):
return "".join(whitespace(term) for term in exp.flat)
class TypeDeclaration(Node):
class TypeDeclaration(Node):
name = property(lambda self: self.type_id[0])
utype = property(lambda self: self.underlying_type.any().any())
type = property(lambda self: self.utype[0] if isinstance(self.utype, list) else self.utype)
@@ -245,7 +246,8 @@ class NamedType(Node):
def do_try(fn):
try:
return fn()
except: pass
except:
pass
def get_rule_id(x):
@@ -255,8 +257,14 @@ def get_rule_id(x):
if matches:
return matches[0]
rule_dependencies = {
k: list(map(operator.attrgetter('contents'), bootstrap.reduce(lambda x, y: x | y, (bootstrap.find_bytype(e, bootstrap.Keyword) for e in [v])))) \
k: list(
map(
operator.attrgetter("contents"),
bootstrap.reduce(lambda x, y: x | y, (bootstrap.find_bytype(e, bootstrap.Keyword) for e in [v])),
)
)
for k, v in bootstrap.express
}
@@ -264,16 +272,17 @@ all_rules = [k for k, e in bootstrap.express]
rule_definitions = {k: v for k, v in bootstrap.express}
def to_tree(x, key=None):
def prune(di):
# translate class names back to grammar rules if nested actions are encountered
di = {get_rule_id(k) or k: v for k, v in di.items()}
def replace_synonyms(x):
for y in x:
yield y
if False: # y in di:
if False: # y in di:
# production element from grammar is found in parsed data,
# return that.
@@ -292,19 +301,21 @@ def to_tree(x, key=None):
yield S
# Do this recursively
yield from replace_synonyms([S])
# is this a concatenation with zero or more synonyms? then also processs that
# @todo catches:
# - simple_expression = term { add_like_op term } .
# but should probably also work on
# - a = b { b }
# in which case the second Concat would be eliminated
elif isinstance(rule, bootstrap.Concat) and \
len(rule.contents) == 2 and \
is_synonym(rule.contents[0]) and \
isinstance(rule.contents[1].contents, bootstrap.Repeated) and \
isinstance(rule.contents[1].contents.contents[0], bootstrap.Concat) and \
str(rule.contents[1].contents.contents[0].contents[1]) == str(rule.contents[0]):
elif (
isinstance(rule, bootstrap.Concat)
and len(rule.contents) == 2
and is_synonym(rule.contents[0])
and isinstance(rule.contents[1].contents, bootstrap.Repeated)
and isinstance(rule.contents[1].contents.contents[0], bootstrap.Concat)
and str(rule.contents[1].contents.contents[0].contents[1]) == str(rule.contents[0])
):
S = is_synonym(rule.contents[0])
yield S
# Do this recursively
@@ -315,13 +326,13 @@ def to_tree(x, key=None):
if key == "aggregation_types":
# hack hack hack apparently the parser can't distinguish these
subrules += list(replace_synonyms(rule_dependencies["general_aggregation_types"]))
if rule_dependencies[key] and not subrules:
# sometimes an intermediate production rule is missing
# from the pyparsing output, e.g from parameter to simple_expression
# directly. Recover from this.
subrules = sum(map(rule_dependencies.__getitem__, rule_dependencies[key]), [])
if not isinstance(rule_definitions[key], bootstrap.Union):
# Filter out terminals when not a union. E.g no
# reason to retain TYPE, END_TYPE, but operators
@@ -331,7 +342,7 @@ def to_tree(x, key=None):
vs = list(di.values())
return {k: v for k, v in di.items() if k in subrules or (k == key and len(vs) == 1 and vs[0] not in all_rules)}
def simplify(di):
if isinstance(di, list):
if set(map(type, di)) == {str} and set(map(len, di)) == {1}:
@@ -343,11 +354,11 @@ def to_tree(x, key=None):
return {k: simplify(v) for k, v in di.items()}
else:
return di
if isinstance(x, ListNode):
d = to_tree(x.dict_tokens, key=get_rule_id(x) or key)
if key == 'if_stmt':
if key == "if_stmt":
# The definition of if statement if (roughy):
# 'if' expr 'then' stmt+ 'else' stmt+
# this causes stmt to be joined under the same
@@ -355,39 +366,41 @@ def to_tree(x, key=None):
# `else_stmt` that collects the second group
# of stmts.
statements = x.dict_tokens['stmt']
statements = x.dict_tokens["stmt"]
else_index = None
if_nesting = 0
for i, tk in enumerate(x.flat):
if tk == 'if': if_nesting += 1
if tk == 'end_if': if_nesting -= 1
if tk == 'else' and if_nesting == 1:
for i, tk in enumerate(x.flat):
if tk == "if":
if_nesting += 1
if tk == "end_if":
if_nesting -= 1
if tk == "else" and if_nesting == 1:
else_index = i
if else_index:
indices = []
for s in statements:
for i in range(max(indices, default=0), len(x.flat)):
if x.flat[i:i+len(s.flat)] == s.flat:
if x.flat[i : i + len(s.flat)] == s.flat:
indices.append(i)
break
assert len(indices) == len(statements)
before_else = [i < else_index for i in indices]
else_stmt = [st for b, st in zip(before_else, d['stmt']) if not b]
d['stmt'] = [st for b, st in zip(before_else, d['stmt']) if b]
else_stmt = [st for b, st in zip(before_else, d["stmt"]) if not b]
d["stmt"] = [st for b, st in zip(before_else, d["stmt"]) if b]
if else_stmt:
d['else_stmt'] = else_stmt
if key == 'formal_parameter':
d["else_stmt"] = else_stmt
if key == "formal_parameter":
# Not so pretty hack to fix the overwriting of simple_id-like
# ast nodes. The full solution would probably to register parse
# actions. And directly reassign.
pid = d['parameter_id'][0][0]
d['parameter_id'][0] = x.flat[:x.flat.index(pid)+1:2]
pid = d["parameter_id"][0][0]
d["parameter_id"][0] = x.flat[: x.flat.index(pid) + 1 : 2]
if key is None:
return {get_rule_id(x): d}
@@ -400,7 +413,10 @@ def to_tree(x, key=None):
elif isinstance(x, dict):
# d = {k: to_tree(v, key=k) for k, v in x.items()}
# not fully understood, but when finding specific node Types and production rules, prioritize the former
d = {get_rule_id(k) or k: to_tree(v, key=k) for k, v in sorted(x.items(), key=lambda p: get_rule_id(p[0]) is not None)}
d = {
get_rule_id(k) or k: to_tree(v, key=k)
for k, v in sorted(x.items(), key=lambda p: get_rule_id(p[0]) is not None)
}
return simplify(prune(d))
elif isinstance(x, list):
return [to_tree(v, key=key) for v in x]
@@ -459,7 +475,8 @@ class SuperTypeExpression(Node):
else:
constraint = self.supertype_rule[0]
return [
list(list(s)[0])[0].simple_id for s in list(list(list(constraint.subtype_constraint[0].supertype_expression[0])[0])[0].one_of[0])[2::2]
list(list(s)[0])[0].simple_id
for s in list(list(list(constraint.subtype_constraint[0].supertype_expression[0])[0])[0].one_of[0])[2::2]
]
sub_types = property(get_sub_types)
@@ -576,10 +593,11 @@ class ProcedureDeclaration(ListNode):
@property
def name(self):
return self.flat[1]
class FunctionDeclaration(ProcedureDeclaration):
pass
class RuleDeclaration(ProcedureDeclaration):
pass