commit: 9113b73dfb0b5c2f7c8f0eb360182991440bba1f
parent 109dc5f2776afb4ece47e07fee951058a4782cd7
Author: Drew DeVault <sir@cmpwn.com>
Date: Sat, 26 Nov 2022 14:55:14 +0100
Hare codegen v2
Diffstat:
1 file changed, 291 insertions(+), 0 deletions(-)
diff --git a/content/blog/Hare-codegen-v2.md b/content/blog/Hare-codegen-v2.md
@@ -0,0 +1,291 @@
+---
+title: Codegen in Hare v2
+date: 2022-11-26
+---
+
+I spoke about code generation in Hare [back in May][0] when I wrote a tool for
+generating ioctl numbers. I wrote another code generator over the past few
+weeks, and it seems like a good time to revisit the topic on my blog to showcase
+another approach, and the improvements we've made for this use-case.
+
+[0]: https://drewdevault.com/2022/05/14/generating-ioctls.html
+
+In this case, I wanted to generate code to implement IPC (inter-process
+communication) interfaces for my operating system. I have designed a <abbr
+title="domain-specific language">DSL</abbr> for describing these interfaces
+— you can [read the grammar here][1]. This calls for a parser, which is
+another interesting topic for Hare, but I'll set that aside for now and focus on
+the code gen. Assume that, given a file like the following, we can parse it and
+produce an AST:
+
+[1]: https://git.sr.ht/~sircmpwn/ipcgen/tree/master/item/doc/grammar.txt
+
+```
+namespace hello;
+
+interface hello {
+ call say_hello() void;
+ call add(a: uint, b: uint) uint;
+};
+```
+
+The key that makes the code gen approach we're looking at today is the
+introduction of [strings::template][2] to the Hare standard library. This module
+is inspired by a similar feature from Python, [string.Template][3]. An example
+of its usage is provided in Hare's standard library documentation:
+
+[2]: https://docs.harelang.org/strings/template
+[3]: https://docs.python.org/3/library/string.html#template-strings
+
+```hare
+const src = "Hello, $user! Your balance is $$$balance.\n";
+const template = template::compile(src)!;
+defer template::finish(&template);
+template::execute(&template, os::stdout,
+ ("user", "ddevault"),
+ ("balance", 1000),
+)!; // "Hello, ddevault! Your balance is $1000.
+```
+
+Makes sense? Cool. Let's see how this can be applied to code generation. The
+interface shown above compiles to the following generated code:
+
+```hare
+// This file was generated by ipcgen; do not modify by hand
+use errors;
+use helios;
+use rt;
+
+def HELLO_ID: u32 = 0xC01CAAC5;
+
+export type fn_hello_say_hello = fn(object: *hello) void;
+export type fn_hello_add = fn(object: *hello, a: uint, b: uint) uint;
+
+export type hello_iface = struct {
+ say_hello: *fn_hello_say_hello,
+ add: *fn_hello_add,
+};
+
+export type hello_label = enum u64 {
+ SAY_HELLO = HELLO_ID << 16u64 | 1,
+ ADD = HELLO_ID << 16u64 | 2,
+};
+
+export type hello = struct {
+ _iface: *hello_iface,
+ _endpoint: helios::cap,
+};
+
+export fn hello_dispatch(
+ object: *hello,
+) void = {
+ const (tag, a1) = helios::recvraw(object._endpoint);
+ switch (rt::label(tag): hello_label) {
+ case hello_label::SAY_HELLO =>
+ object._iface.say_hello(
+ object,
+ );
+ match (helios::reply(0)) {
+ case void =>
+ yield;
+ case errors::invalid_cslot =>
+ yield; // callee stored the reply
+ case errors::error =>
+ abort(); // TODO
+ };
+ case hello_label::ADD =>
+ const rval = object._iface.add(
+ object,
+ a1: uint,
+ rt::ipcbuf.params[1]: uint,
+ );
+ match (helios::reply(0, rval)) {
+ case void =>
+ yield;
+ case errors::invalid_cslot =>
+ yield; // callee stored the reply
+ case errors::error =>
+ abort(); // TODO
+ };
+ case =>
+ abort(); // TODO
+ };
+};
+```
+
+Generating this code starts with the following entry-point:
+
+```hare
+// Generates code for a server to implement the given interface.
+export fn server(out: io::handle, doc: *ast::document) (void | io::error) = {
+ fmt::fprintln(out, "// This file was generated by ipcgen; do not modify by hand")!;
+ fmt::fprintln(out, "use errors;")!;
+ fmt::fprintln(out, "use helios;")!;
+ fmt::fprintln(out, "use rt;")!;
+ fmt::fprintln(out)!;
+
+ for (let i = 0z; i < len(doc.interfaces); i += 1) {
+ const iface = &doc.interfaces[i];
+ s_iface(out, doc, iface)?;
+ };
+};
+```
+
+Here we start with some simple use of basic string formatting via
+[fmt::fprintln][fmt::fprintln]. We see some of the same approach repeated in the
+meatier functions like s\_iface:
+
+[fmt::fprintln]: https://docs.harelang.org/fmt#fprintln
+
+```hare
+fn s_iface(
+ out: io::handle,
+ doc: *ast::document,
+ iface: *ast::interface,
+) (void | io::error) = {
+ const id: ast::ident = [iface.name];
+ const name = gen_name_upper(&id);
+ defer free(name);
+
+ let id: ast::ident = alloc(doc.namespace...);
+ append(id, iface.name);
+ defer free(id);
+ const hash = genhash(&id);
+
+ fmt::fprintfln(out, "def {}_ID: u32 = 0x{:X};\n", name, hash)!;
+```
+
+Our first use of strings::template appears when we want to generate type aliases
+for interface functions, via s\_method\_fntype. This is where some of the
+trade-offs of this approach begin to present themselves.
+
+```hare
+const s_method_fntype_src: str =
+ `export type fn_$iface_$method = fn(object: *$object$params) $result;`;
+let st_method_fntype: tmpl::template = [];
+
+@init fn s_method_fntype() void = {
+ st_method_fntype= tmpl::compile(s_method_fntype_src)!;
+};
+
+fn s_method_fntype(
+ out: io::handle,
+ iface: *ast::interface,
+ meth: *ast::method,
+) (void | io::error) = {
+ assert(len(meth.caps_in) == 0); // TODO
+ assert(len(meth.caps_out) == 0); // TODO
+
+ let params = strio::dynamic();
+ defer io::close(¶ms)!;
+ if (len(meth.params) != 0) {
+ fmt::fprint(¶ms, ", ")?;
+ };
+ for (let i = 0z; i < len(meth.params); i += 1) {
+ const param = &meth.params[i];
+ fmt::fprintf(¶ms, "{}: ", param.name)!;
+ ipc_type(¶ms, ¶m.param_type)!;
+
+ if (i + 1 < len(meth.params)) {
+ fmt::fprint(¶ms, ", ")!;
+ };
+ };
+
+ let result = strio::dynamic();
+ defer io::close(&result)!;
+ ipc_type(&result, &meth.result)!;
+
+ tmpl::execute(&st_method_fntype, out,
+ ("method", meth.name),
+ ("iface", iface.name),
+ ("object", iface.name),
+ ("params", strio::string(¶ms)),
+ ("result", strio::string(&result)),
+ )?;
+ fmt::fprintln(out)?;
+};
+```
+
+The simple string substitution approach of strings::template prevents it from
+being as generally useful as a full-blown templating engine ala jinja2. To work
+around this, we have to write Hare code which does things like slurping up the
+method parameters into a [strio::dynamic] buffer where we might instead reach
+for something like
+<code>{% for param in method.params %}</code> in
+jinja2. Once we have prepared all of our data in a format suitable for a linear
+string substitution, we can pass it to
+<abbr title="This file aliases strings::template as tmpl to simplify things a bit.">tmpl</abbr>::execute.
+The actual template is stored in a global which is compiled during @init, which
+runs at program startup. Anything which requires a loop to compile, such as the
+parameter list, is fetched out of the strio buffer and passed to the template.
+
+[strio::dynamic]: https://docs.harelang.org/strio#dynamic
+
+We can explore a slightly different approach when we generate this part of the
+code, back up in the s\_iface function:
+
+```hare
+export type hello_iface = struct {
+ say_hello: *fn_hello_say_hello,
+ add: *fn_hello_add,
+};
+```
+
+To output this code, we render several templates one after another, rather than
+slurping up the generated code into heap-allocated string buffers to be passed
+into a single template.
+
+```hare
+const s_iface_header_src: str =
+ `export type $iface_iface = struct {`;
+let st_iface_header: tmpl::template = [];
+
+const s_iface_method_src: str =
+ ` $method: *fn_$iface_$method,`;
+let st_iface_method: tmpl::template = [];
+
+@init fn s_iface() void = {
+ st_iface_header = tmpl::compile(s_iface_header_src)!;
+ st_iface_method = tmpl::compile(s_iface_method_src)!;
+};
+
+// ...
+
+tmpl::execute(&st_iface_header, out,
+ ("iface", iface.name),
+)?;
+fmt::fprintln(out)?;
+
+for (let i = 0z; i < len(iface.methods); i += 1) {
+ const meth = &iface.methods[i];
+ tmpl::execute(&st_iface_method, out,
+ ("iface", iface.name),
+ ("method", meth.name),
+ )?;
+ fmt::fprintln(out)?;
+};
+
+fmt::fprintln(out, "};\n")?;
+```
+
+The [remainder of the code][4] is fairly similar.
+
+[4]: https://git.sr.ht/~sircmpwn/ipcgen/tree/2cdc53095a052b4f5ce3fdc6e410f2dd17eea54d/item/gen/server.ha
+
+strings::template is less powerful than a more sophisticated templating system
+might be, such as Golang's text/template. A more sophisticated templating engine
+could be implemented for Hare, but it would be more challenging — no
+reflection or generics in Hare — and would not be a great candidate for
+the standard library. This approach hits the sweet spot of simplicity and
+utility that we're aiming for in the Hare stdlib. strings::template is
+implemented in [a single ~180 line file][5].
+
+[5]: https://git.sr.ht/~sircmpwn/hare/tree/da003e45ced4991b1bae282169dcf942e1e4b235/item/strings/template/template.ha
+
+I plan to continue polishing this tool so I can use it to describe interfaces
+for communications between userspace drivers and other low-level userspace
+services in my operating system. If you have any questions, feel free to post
+them on my public inbox, or shoot them over to my new [fediverse account][6].
+Until next time!
+
+[6]: https://fosstodon.org/@drewdevault
