From bff6a894e01c07af0e465995e2b4e21aefa7472e Mon Sep 17 00:00:00 2001 From: Not Zed Date: Mon, 28 Jul 2025 10:19:16 +0930 Subject: [PATCH] Work on new Java based generator --- .../classes/au/notzed/nativez/tools/API.java | 703 ++++++++++++++++++ .../au/notzed/nativez/tools/Array.java | 59 ++ .../au/notzed/nativez/tools/Context.java | 445 +++++++++++ .../classes/au/notzed/nativez/tools/Data.java | 84 +++ .../au/notzed/nativez/tools/Export.java | 222 ++++++ .../au/notzed/nativez/tools/Generator.java | 180 +++++ .../classes/au/notzed/nativez/tools/Hash.java | 118 +++ .../au/notzed/nativez/tools/Scalar.java | 51 ++ .../au/notzed/nativez/tools/Template.java | 406 ++++++++++ .../classes/au/notzed/nativez/tools/Text.java | 69 ++ .../au/notzed/nativez/tools/Tokeniser.java | 441 +++++++++++ .../classes/au/notzed/nativez/tools/Util.java | 80 ++ .../au/notzed/nativez/tools/Value.java | 108 +++ .../classes/module-info.java | 19 + 14 files changed, 2985 insertions(+) create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/API.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Array.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Context.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Data.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Export.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Generator.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Hash.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Scalar.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Template.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Text.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Tokeniser.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Util.java create mode 100644 src/notzed.nativez.tools/classes/au/notzed/nativez/tools/Value.java create mode 100644 src/notzed.nativez.tools/classes/module-info.java diff --git a/src/notzed.nativez.tools/classes/au/notzed/nativez/tools/API.java b/src/notzed.nativez.tools/classes/au/notzed/nativez/tools/API.java new file mode 100644 index 0000000..287266a --- /dev/null +++ b/src/notzed.nativez.tools/classes/au/notzed/nativez/tools/API.java @@ -0,0 +1,703 @@ +/* + * Copyright (C) 2025 Michael Zucchi + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package au.notzed.nativez.tools; + +import java.io.IOException; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** + * Loads and processes an .api file. + *

+ */ +public class API { + + /* data loaded from api file */ + List libraries; + Deque types; + // Expanded code-map with key "name:field" + Map codes; + // Expanded system-map with key 'type:name'? Is this useful? + Map system; + + public API(List system, List libraries, Deque types, Collection codes) { + this.libraries = libraries; + this.types = types; + this.codes = new HashMap<>(); + this.system = new HashMap<>(); + + // Codes are ordered in reverse, only take first entry to override others + for (var c: codes) { + for (var e: c.fields.entrySet()) { + this.codes.putIfAbsent(c.name + ":" + e.getKey(), e.getValue()); + } + } + for (var c: system) { + for (var e: c.system.entrySet()) { + this.system.put(c.name + ":" + e.getKey(), e.getValue()); + } + } + } + + public Data code() { + return Data.ofTextMap(codes); + } + + public Data code(String module) { + return Data.ofTextMap(codes, module + ":"); + } + + public Data system() { + return Data.ofTextMap(system); + } + + public Data system(String part) { + return Data.ofTextMap(system, part + ":"); + } + + public Text getCode(String type, String name) { + return codes.getOrDefault(type + ":" + name, Text.UNDEFINED); + } + + public Text getCode(Text key) { + switch (key.type()) { + case LITERAL: + case STRING: + return key; + case IDENTIFIER: + return codes.getOrDefault(key.value(), Text.UNDEFINED); + default: + throw new NoSuchElementException("No key for :" + key); + } + } + + public Text getSystem(Match.Type type, String key, Text fallback) { + var v = system.get(type.name() + ":" + key); + if (v != null) + return v; + else + return fallback; + } + + public DependencyMap newDependencyMap(Export capi) { + return new DependencyMap(capi); + } + + public class DependencyMap { + Set seen = new HashSet<>(); + Export capi; + // Stores expadned types + Map visited = new HashMap<>(); + + public DependencyMap(Export capi) { + this.capi = capi; + } + + public Stream visisted(Match.Type type) { + String match = type.name() + ":"; + return seen.stream().filter(s -> s.startsWith(match)).map(capi::getType); + } + + // Maybe return all types in set (i.e.g api.visited) plus anon types (merge?) + public void visitDependencies(Set roots) { + Deque queue = new ArrayDeque<>(roots.size()); + + // Types from the start set + for (String s: roots) { + Hash h = capi.getType(s); + assert h.defined(); + if (seen.add(s)) + queue.add(h); + } + /* + for (Value v: start.hash.values()) { + Hash h = (Hash)v; + System.out.println(h.getScalar("type").getValue() + ":" + h.getScalar("name").getValue()); + seen.add(h.getScalar("type").getValue() + ":" + h.getScalar("name").getValue()); + queue.add((Hash)v); + }*/ + + while (!queue.isEmpty()) { + Hash type = queue.removeFirst(); + + //System.out.println("\nprocess: " + type.getScalar("type").value + type.getScalar("name").value); + + //type.dump(System.out); + //Export.allFields(type).forEach(h -> h.dump(System.out)); + + Export.allFields(type) + .map(Hash.field("deref")) + .forEach(this::visitType); + + // What do i add to the queue? + // What about all the types it added, how do look them up? + // Field 'type' will map t capi->{key} + Export.allFields(type) + .map(Hash.field("type")) + .filter(s -> s.indexOf(':') >= 0) + .filter(seen::add) + .filter(s -> { + Hash v = capi.types.getHash(s); + if (!v.defined()) { + throw new RuntimeException("no type: " + s); + } + /* + if (!v.defined()) { + if (s.startsWith("struct:")) { + v = new Hash(); + v.put("type", s); + } else { + System.getLogger(API.class.getName()).log(System.Logger.Level.WARNING, () -> "Unsupported anonymous type: " + s); + } + }*/ + return v.defined(); + }) + .map(s -> capi.types.getHash(s)) + .forEach(queue::add); + } + } + + /** + * Mark a type as used. + * This is used to build a concrete type map as well as indicate which types are + * in use. + * + * @param deref Expanded raw type name. + */ + public void visitType(String deref) { + Matcher matcher = null; + TemplateMap matched = null; + + //System.out.printf("visit: %s\n", deref); + + if (visited.containsKey(deref)) + return; + + // Find the last matching type, exact matches override pattern matching. + for (var type: types) { + if (type.exact(deref)) { + System.out.println("add exact type: " + deref); + visited.put(deref, type); + return; + } else if (matched == null) { + Matcher test = type.pattern.matcher(deref); + + if (test.matches()) { + matcher = test; + matched = type; + } + } + } + + if (matcher != null && matched != null) { + TemplateMap type = matched.copyAs(deref); + + for (var e: matcher.namedGroups().entrySet()) + type.fields.put(e.getKey(), Text.ofLiteral(matcher.group(e.getValue()))); + + System.out.println("add matching type: " + deref); + visited.put(deref, type); + } else { + throw new NoSuchElementException("No match for type: " + deref); + } + } + + public Function typeMap() { + // Copy all flags from types to instances of/pointers to types + seen.stream().filter(s -> s.startsWith("struct:")).map(capi::getType).forEach(type -> { + String name = type.getScalar("name").value; + TemplateMap temp = visited.get("u64:${" + name + "}"); + if (temp != null) { + type.hash.entrySet().stream().filter(e -> e.getKey().endsWith("?")) + .forEach(e -> { + temp.setTemplate(e.getKey(), Text.ofValue(e.getValue())); + }); + } else { + System.getLogger(API.class.getName()).log(System.Logger.Level.INFO, () -> "No pointer for anonymous type: " + name); + } + }); + + return (key) -> { + return visited.get(key); + }; + } + } + + /** + * For storing code and type blocks. + *

+ * TODO: templates can be pre-compiled, do that here on-demand? + */ + static class TemplateMap implements Data { + String name; + Pattern pattern; + Map fields = new HashMap<>(); + + public TemplateMap(String name, boolean regex) { + this.name = name; + this.pattern = regex ? Pattern.compile(name) : null; + } + + public TemplateMap(String name) { + this(name, false); + } + + public TemplateMap copyAs(String name) { + TemplateMap map = new TemplateMap(name); + map.fields.putAll(fields); + return map; + } + + public boolean exact(String name) { + return pattern == null && this.name.equals(name); + } + + public void setTemplate(String name, Text template) { + fields.put(name, template); + } + + // pre-parse here? + public Text getTemplate(String name) { + return fields.get(name); + } + + public void inherit(TemplateMap b) { + fields.putAll(b.fields); + } + + @Override + public Value getValue(Text key) { + return getText(key).toValue(); + } + + @Override + public Text getText(Text key) { + return fields.getOrDefault(key.value(), Text.UNDEFINED); + } + + public void dump(PrintStream out) { + out.printf("\ttype: %s\n", name); + for (var e: fields.entrySet()) { + out.printf("\t\t%s", e.getKey()); + switch (e.getValue().type()) { + case IDENTIFIER -> + out.printf(" %s\n", e.getValue().value()); + case LITERAL -> + out.printf(" <[%s]>\n", e.getValue().value()); + case STRING -> + out.printf(" \"%s\";\n", e.getValue().value()); + } + } + } + } + + static record Option(String name, Text code) { + Option(Text code) { + this("#flag", code); + } + } + + static record Match(Type type, Predicate matcher, List

+ */ + static class LibraryInfo { + String name; + List load = new ArrayList<>(); + // non-pattern fields + Map system = new HashMap<>(); + // Patterns to match things to templates + List field = new ArrayList<>(); + + public LibraryInfo(String name) { + this.name = name; + } + + // Use system() ? + public Text getSystem(Match.Type type, String key, Text fallback) { + var v = system.get(type.name() + ":" + key); + if (v != null) + return v; + else + return fallback; + } + + public Data system() { + return Data.ofTextMap(system); + } + + public Data system(String type) { + return Data.ofTextMap(system, type + ":"); + } + + void dump(PrintStream out) { + out.printf("library: %s\n", name); + for (var l: load) + out.printf(" load %s\n", l); + for (var m: field) { + out.printf(" %s\n", m); + } + } + + } + + static class APIParser implements AutoCloseable { + static final char[] assoc = {'=', '>'}; + private final Tokeniser in; + private final Path base; + + List libraries = new ArrayList<>(); + List system = new ArrayList<>(); + Deque typeClasses = new ArrayDeque<>(); + Deque types = new ArrayDeque<>(); + Deque codeClasses = new ArrayDeque<>(); + Deque codes = new ArrayDeque<>(); + + public APIParser(Path base, Tokeniser in) { + this.in = in; + this.base = base; + } + + enum Types { + code, + type, + include, + library, + system; + } + + Types getType(String key) throws IOException { + try { + return Types.valueOf(key); + } catch (IllegalArgumentException x) { + throw new IOException(in.fatalMessage("Invalid object type: " + key)); + } + } + + @Override + public void close() throws IOException { + in.close(); + } + + String identifier(int t) throws IOException { + if (t != Tokeniser.TOK_IDENTIFIER) + throw new IOException(in.fatalMessage(t, "Expeccting identifier")); + return in.getText(); + } + + /** + * Read a list of strings until ';'. + * Strings are string or identifier. + * + * @return + * @throws IOException + */ + List stringList() throws IOException { + List list = new ArrayList<>(); + int t; + while ((t = in.nextToken()) != -1 && t != ';') { + switch (t) { + case Tokeniser.TOK_IDENTIFIER: + case Tokeniser.TOK_STRING_SQ: + case Tokeniser.TOK_STRING_DQ: + list.add(in.getText()); + break; + default: + throw new IOException(in.fatalMessage(t, "Expecting STRING IDENTIFIER or ';'")); + } + } + return list; + } + + /** + * Looks for ';' and nothing else. + * + * @throws IOException + */ + void emptyList() throws IOException { + int t = in.nextToken(); + if (t != ';') + throw new IOException(in.fatalMessage(t, "Expecting ';'")); + } + + /** + * Load a 'type' specification. + * Applies inheritance as it goes. + * + * @throws IOException + */ + TemplateMap loadType(Deque classes, Deque types) throws IOException { + TemplateMap map; + int t; + String text; + + switch ((t = in.nextToken())) { + case Tokeniser.TOK_REGEX: + text = in.getText(); + map = new TemplateMap(in.getText(), true); + types.add(map); + break; + case Tokeniser.TOK_IDENTIFIER: + map = new TemplateMap(in.getText()); + classes.addFirst(map); + break; + default: + throw new IOException(in.fatalMessage(t, "Expected name or regex")); + } + + // options before {, inherit + // do i need options? or just inherit? + while ((t = in.nextToken()) != -1 && t != '{') { + String tmp; + + switch (t) { + case Tokeniser.TOK_IDENTIFIER: + tmp = in.getText(); + classes.stream().filter(c -> c.exact(tmp)).findFirst().ifPresent(map::inherit); + break; + case ';': + return map; + default: + throw new IOException(in.fatalMessage(t, "Expected name")); + } + } + + // Parse fields + while ((t = in.nextToken()) != -1 && t != '}') { + String key = identifier(t); + + // TODO: support options, do i need options? + Text value = textValue(); + + map.setTemplate(key, value); + } + return map; + } + + Text textValue() throws IOException { + int c = in.nextToken(); + Text text = in.getText(c); + + if (c == Tokeniser.TOK_QUOTED || c == Tokeniser.TOK_LITERAL) + return text; + + c = in.nextToken(); + if (c != ';') + throw new IOException(in.fatalMessage(c, "Expected ';'")); + return text; + } + + Text textOrNoValue() throws IOException { + int c = in.nextToken(); + + if (c == ';') + return Text.ofLiteral("#true"); + + Text text = in.getText(c); + + if (c == Tokeniser.TOK_QUOTED || c == Tokeniser.TOK_LITERAL) + return text; + + c = in.nextToken(); + if (c != ';') + throw new IOException(in.fatalMessage(c, "Expected ';'")); + return text; + } + + List