Interpreter started!

This commit is contained in:
Robert Bendun 2022-05-16 02:18:53 +02:00
parent 14ea494686
commit ccd2166231
10 changed files with 355 additions and 27 deletions

View File

@ -1,14 +1,17 @@
MAKEFLAGS="-j $(grep -c ^processor /proc/cpuinfo)"
CXXFLAGS:=$(CXXFLAGS) -std=c++20 -Wall -Wextra -Werror=switch -Werror=unused-result
CXXFLAGS:=$(CXXFLAGS) -std=c++20 -Wall -Wextra -Werror=switch -Werror=return-type -Werror=unused-result
CPPFLAGS:=$(CPPFLAGS) -Ilib/expected/ -Ilib/ut/ -Isrc/
Obj=bin/errors.o \
Obj= \
bin/errors.o \
bin/interpreter.o \
bin/lexer.o \
bin/location.o \
bin/number.o \
bin/parser.o \
bin/unicode.o \
bin/number.o \
bin/unicode_tables.o
bin/unicode_tables.o \
bin/value.o
all: bin/musique bin/unit-tests

3
examples/arithmetic.mq Normal file
View File

@ -0,0 +1,3 @@
say (1 + 3);
say (3 * 10);
say 42

View File

@ -69,8 +69,9 @@ std::ostream& operator<<(std::ostream& os, Error const& err)
case errors::Unrecognized_Character:
return err.message.empty() ? os << "unrecognized character\n" : os << err.message;
case errors::Function_Not_Defined:
case errors::Unexpected_Token_Type:
return os << err.message;
return os << err.message << '\n';
case errors::Unexpected_Empty_Source:
return os << "unexpected end of input\n";
@ -144,6 +145,14 @@ Error errors::failed_numeric_parsing(Location location, std::errc errc, std::str
return err;
}
Error errors::function_not_defined(Value const& value)
{
Error err;
err.type = Function_Not_Defined;
err.message = "Function '" + value.s + "' has not been defined yet";
return err;
}
void errors::all_tokens_were_not_parsed(std::span<Token> tokens)
{
error_heading(std::cerr, std::nullopt, Error_Level::Bug);

96
src/interpreter.cc Normal file
View File

@ -0,0 +1,96 @@
#include <musique.hh>
#include <iostream>
Interpreter::Interpreter()
: Interpreter(std::cout)
{
}
Interpreter::Interpreter(std::ostream& out)
: out(out)
{
functions["typeof"] = [](std::vector<Value> args) -> Value {
assert(args.size() == 1, "typeof expects only one argument");
switch (args.front().type) {
case Value::Type::Nil: return Value::symbol("nil");
case Value::Type::Number: return Value::symbol("number");
case Value::Type::Symbol: return Value::symbol("symbol");
}
unreachable();
};
functions["say"] = [&](std::vector<Value> args) -> Value {
for (auto it = args.begin(); it != args.end(); ++it) {
out << *it;
if (std::next(it) != args.end())
out << ' ';
}
out << '\n';
return {};
};
}
Result<Value> Interpreter::eval(Ast &&ast)
{
switch (ast.type) {
case Ast::Type::Literal:
return Value::from(ast.token);
case Ast::Type::Binary:
{
std::vector<Value> values;
values.reserve(ast.arguments.size());
for (auto& a : ast.arguments) {
values.push_back(Try(eval(std::move(a))));
}
Value result = values.front();
if (ast.token.source == "+") {
for (auto &v : std::span(values).subspan(1)) {
assert(result.type == Value::Type::Number, "LHS of + should be a number");
assert(v.type == Value::Type::Number, "RHS of + should be a number");
result.n += v.n;
}
return result;
} else if (ast.token.source == "*") {
for (auto &v : std::span(values).subspan(1)) {
assert(result.type == Value::Type::Number, "LHS of * should be a number");
assert(v.type == Value::Type::Number, "RHS of * should be a number");
result.n *= v.n;
}
return result;
}
unimplemented();
}
break;
case Ast::Type::Sequence:
{
Value v;
for (auto &a : ast.arguments)
v = Try(eval(std::move(a)));
return v;
}
case Ast::Type::Call:
{
Value func = Try(eval(std::move(ast.arguments.front())));
assert(func.type == Value::Type::Symbol, "Currently only symbols can be called");
if (auto it = functions.find(func.s); it != functions.end()) {
std::vector<Value> values;
values.reserve(ast.arguments.size());
for (auto& a : std::span(ast.arguments).subspan(1)) {
values.push_back(Try(eval(std::move(a))));
}
return it->second(std::move(values));
} else {
return errors::function_not_defined(func);
}
}
default:
unimplemented();
}
}

View File

@ -7,6 +7,8 @@
namespace fs = std::filesystem;
static bool ast_only_mode = false;
static std::string_view pop(std::span<char const*> &span)
{
auto element = span.front();
@ -23,17 +25,25 @@ void usage()
" where options are:\n"
" -c CODE\n"
" --run CODE\n"
" executes given code\n";
" executes given code\n"
"\n"
" --ast\n"
" prints ast for given code\n";
std::exit(1);
}
static Result<void> run(std::string_view source, std::string_view filename)
{
auto ast = Try(Parser::parse(source, filename));
std::cout << "successfully parsed: " << source << " \n";
if (ast_only_mode) {
dump(ast);
return {};
}
Interpreter interpreter;
std::cout << Try(interpreter.eval(std::move(ast))) << std::endl;
return {};
}
// We make sure that through life of interpreter source code is allways allocated
std::vector<std::string> eternal_sources;
@ -44,12 +54,13 @@ static Result<void> Main(std::span<char const*> args)
usage();
}
bool runned_something = false;
std::vector<std::string_view> files;
while (not args.empty()) {
std::string_view arg = pop(args);
if (!arg.starts_with('-')) {
if (arg == "-" || !arg.starts_with('-')) {
files.push_back(std::move(arg));
continue;
}
@ -62,6 +73,12 @@ static Result<void> Main(std::span<char const*> args)
auto const source = pop(args);
Try(run(source, "arguments"));
runned_something = true;
continue;
}
if (arg == "--ast") {
ast_only_mode = true;
continue;
}
@ -69,13 +86,18 @@ static Result<void> Main(std::span<char const*> args)
std::exit(1);
}
if (!runned_something && files.empty()) {
usage();
}
for (auto const& path : files) {
if (path == "-") {
eternal_sources.emplace_back(std::istreambuf_iterator<char>(std::cin), std::istreambuf_iterator<char>());
} else {
if (not fs::exists(path)) {
std::cerr << "musique: error: couldn't open file: " << path << std::endl;
std::exit(1);
}
{
std::ifstream source_file{fs::path(path)};
eternal_sources.emplace_back(std::istreambuf_iterator<char>(source_file), std::istreambuf_iterator<char>());
}

View File

@ -44,6 +44,8 @@ namespace errors
Unexpected_Token_Type,
Unexpected_Empty_Source,
Failed_Numeric_Parsing,
Function_Not_Defined
};
}
@ -380,6 +382,49 @@ struct Number
std::ostream& operator<<(std::ostream& os, Number const& num);
// TODO Add location
struct Value
{
static Result<Value> from(Token t);
static Value number(Number n);
static Value symbol(std::string s);
enum class Type
{
Nil,
Number,
Symbol,
};
Type type = Type::Nil;
Number n{};
// TODO Most strings should not be allocated by Value, but reference to string allocated previously
// Wrapper for std::string is needed that will allocate only when needed, middle ground between:
// std::string - always owning string type
// std::string_view - not-owning string type
std::string s{};
bool operator==(Value const& other) const;
};
std::ostream& operator<<(std::ostream& os, Value const& v);
using Function = std::function<Value(std::vector<Value>)>;
struct Interpreter
{
std::ostream &out;
std::unordered_map<std::string, Function> functions;
Interpreter();
Interpreter(std::ostream& out);
Interpreter(Interpreter const&) = delete;
Interpreter(Interpreter &&) = default;
Result<Value> eval(Ast &&ast);
};
namespace errors
{
Error unrecognized_character(u32 invalid_character);
@ -391,6 +436,8 @@ namespace errors
Error failed_numeric_parsing(Location location, std::errc errc, std::string_view source);
Error function_not_defined(Value const& v);
[[noreturn]]
void all_tokens_were_not_parsed(std::span<Token>);
}

View File

@ -2,7 +2,19 @@
#include <iostream>
static Ast wrap_if_several(std::vector<Ast> &&ast, Ast(*wrapper)(std::vector<Ast>));
static Result<std::vector<Ast>> parse_one_or_more(Parser &p, Result<Ast> (Parser::*parser)(), std::optional<Token::Type> separator = std::nullopt);
enum class At_Least : bool
{
Zero,
One
};
static Result<std::vector<Ast>> parse_many(
Parser &p,
Result<Ast> (Parser::*parser)(),
std::optional<Token::Type> separator,
At_Least at_least);
Result<Ast> Parser::parse(std::string_view source, std::string_view filename)
{
@ -29,7 +41,7 @@ Result<Ast> Parser::parse(std::string_view source, std::string_view filename)
Result<Ast> Parser::parse_sequence()
{
auto seq = Try(parse_one_or_more(*this, &Parser::parse_expression, Token::Type::Expression_Separator));
auto seq = Try(parse_many(*this, &Parser::parse_expression, Token::Type::Expression_Separator, At_Least::Zero));
return wrap_if_several(std::move(seq), Ast::sequence);
}
@ -40,7 +52,7 @@ Result<Ast> Parser::parse_expression()
Result<Ast> Parser::parse_infix_expression()
{
auto atomics = Try(parse_one_or_more(*this, &Parser::parse_atomic_expression));
auto atomics = Try(parse_many(*this, &Parser::parse_atomic_expression, std::nullopt, At_Least::One));
auto lhs = wrap_if_several(std::move(atomics), Ast::call);
if (expect(Token::Type::Operator)) {
@ -71,7 +83,7 @@ Result<Ast> Parser::parse_atomic_expression()
auto start = token_id;
std::vector<Ast> parameters;
if (auto p = parse_one_or_more(*this, &Parser::parse_identifier); p && expect(Token::Type::Variable_Separator)) {
if (auto p = parse_many(*this, &Parser::parse_identifier, std::nullopt, At_Least::One); p && expect(Token::Type::Variable_Separator)) {
consume();
parameters = std::move(p).value();
} else {
@ -108,10 +120,20 @@ Result<Ast> Parser::parse_identifier()
return lit;
}
Result<std::vector<Ast>> parse_one_or_more(Parser &p, Result<Ast> (Parser::*parser)(), std::optional<Token::Type> separator)
static Result<std::vector<Ast>> parse_many(
Parser &p,
Result<Ast> (Parser::*parser)(),
std::optional<Token::Type> separator,
At_Least at_least)
{
std::vector<Ast> trees;
Result<Ast> expr;
if (at_least == At_Least::Zero && p.token_id >= p.tokens.size())
return {};
while ((expr = (p.*parser)()).has_value()) {
trees.push_back(std::move(expr).value());
if (separator) {
@ -197,8 +219,8 @@ Ast Ast::sequence(std::vector<Ast> expressions)
ast.type = Type::Sequence;
if (!expressions.empty()) {
ast.location = expressions.front().location;
}
ast.arguments = std::move(expressions);
}
return ast;
}
@ -280,8 +302,7 @@ struct Indent
std::ostream& operator<<(std::ostream& os, Indent n)
{
while (n.count-- > 0)
os.put(' ');
std::fill_n(std::ostreambuf_iterator<char>(os), n.count, ' ');
return os;
}

61
src/tests/interpreter.cc Normal file
View File

@ -0,0 +1,61 @@
#include <boost/ut.hpp>
#include <musique.hh>
using namespace boost::ut;
using namespace std::string_view_literals;
static auto capture_errors(auto lambda, reflection::source_location sl = reflection::source_location::current())
{
return [=] {
auto result = lambda();
if (not result.has_value()) {
expect(false, sl) << "failed test with error: " << result.error();
}
};
}
void evaluates_to(Value value, std::string_view source_code, reflection::source_location sl = reflection::source_location::current())
{
capture_errors([=]() -> Result<void> {
Interpreter interpreter;
auto result = Try(interpreter.eval(Try(Parser::parse(source_code, "test"))));
expect(eq(result, value));
return {};
}, sl)();
}
void produces_output(std::string_view source_code, std::string_view expected_output, reflection::source_location sl = reflection::source_location::current())
{
capture_errors([=]() -> Result<void> {
std::stringstream ss;
Interpreter interpreter(ss);
Try(interpreter.eval(Try(Parser::parse(source_code, "test"))));
expect(eq(ss.str(), expected_output)) << "different evaluation output";
return {};
}, sl)();
}
suite intepreter_test = [] {
"Interpreter"_test = [] {
should("evaluate literals") = [] {
evaluates_to(Value{}, "nil");
evaluates_to(Value::number(Number(10)), "10");
evaluates_to(Value::symbol("notexistingsymbol"), "notexistingsymbol");
};
should("evaluate arithmetic") = [] {
evaluates_to(Value::number(Number(10)), "5 + 3 + 2");
evaluates_to(Value::number(Number(25)), "5 * (3 + 2)");
};
should("call builtin functions") = [] {
evaluates_to(Value::symbol("nil"), "typeof nil");
evaluates_to(Value::symbol("symbol"), "typeof foo");
evaluates_to(Value::symbol("number"), "typeof 100");
produces_output("say 5", "5\n");
produces_output("say 1; say 2; say 3", "1\n2\n3\n");
produces_output("say 1 2 3", "1 2 3\n");
};
};
};

View File

@ -14,6 +14,10 @@ void expect_ast(
}
suite parser_test = [] {
"Empty file parsing"_test = [] {
expect_ast("", Ast::sequence({}));
};
"Literal parsing"_test = [] {
expect_ast("1", Ast::literal(Token { Token::Type::Numeric, "1", {} }));
};

62
src/value.cc Normal file
View File

@ -0,0 +1,62 @@
#include <musique.hh>
Result<Value> Value::from(Token t)
{
switch (t.type) {
case Token::Type::Numeric:
return Number::from(std::move(t)).map(Value::number);
case Token::Type::Symbol:
if (t.source == "nil")
return Value{};
return Value::symbol(std::string(t.source));
default:
unimplemented();
}
}
Value Value::number(Number n)
{
Value v;
v.type = Type::Number;
v.n = n;
return v;
}
Value Value::symbol(std::string s)
{
Value v;
v.type = Type::Symbol;
v.s = s;
return v;
}
bool Value::operator==(Value const& other) const
{
if (type != other.type)
return false;
switch (type) {
case Type::Nil: return true;
case Type::Number: return n == other.n;
case Type::Symbol: return s == other.s;
}
unreachable();
}
std::ostream& operator<<(std::ostream& os, Value const& v)
{
switch (v.type) {
case Value::Type::Nil:
return os << "nil";
case Value::Type::Number:
return os << v.n;
case Value::Type::Symbol:
return os << v.s;
}
unreachable();
}