From dd403c1ed24ca52b4d89a5d2e6635d1f5f886511 Mon Sep 17 00:00:00 2001 From: Neonwave175 Date: Wed, 18 Nov 2026 22:19:52 +0530 Subject: [PATCH 1/3] Add TellC list mutation and shared operators --- TellC/include/tell/parser.hpp | 2 +- TellC/src/codegen.cpp | 165 ++++++++++++++++-- TellC/src/parser.cpp | 14 +- TellC/tests/compiler_integration.cmake | 4 + TellC/tests/fixtures/mutable_collections.tell | 48 +++++ TellC/tests/fixtures/mutable_target.tell | 1 + TellC/tests/fixtures/mutable_type.tell | 2 + 7 files changed, 221 insertions(+), 15 deletions(-) create mode 100644 TellC/tests/fixtures/mutable_collections.tell create mode 100644 TellC/tests/fixtures/mutable_target.tell create mode 100644 TellC/tests/fixtures/mutable_type.tell diff --git a/TellC/include/tell/parser.hpp b/TellC/include/tell/parser.hpp index 7c6f674..60d2c93 100644 --- a/TellC/include/tell/parser.hpp +++ b/TellC/include/tell/parser.hpp @@ -20,7 +20,7 @@ struct Expr { struct Parameter { std::string type; std::string name; Location location; }; struct ForeignParameter { std::string type; std::string name; Location location; }; struct Stmt { - enum class Kind { Let, Set, Say, Check, Return, Expression, If, ElseIf, Else, Switch, Case, While, Repeat, For, Function, Object, AccessSection, ForeignFunction, Break, Continue, Try, Raise }; + enum class Kind { Let, Set, Add, Remove, Say, Check, Return, Expression, If, ElseIf, Else, Switch, Case, While, Repeat, For, Function, Object, AccessSection, ForeignFunction, Break, Continue, Try, Raise }; Kind kind; Location location; std::string name; diff --git a/TellC/src/codegen.cpp b/TellC/src/codegen.cpp index 615a947..381d974 100644 --- a/TellC/src/codegen.cpp +++ b/TellC/src/codegen.cpp @@ -54,7 +54,7 @@ class Generator { Value receiver{nullptr, ""}; std::vector> loops; bool is_list(const std::string& type) { return type.rfind("list of ", 0) == 0; } - llvm::StructType* list_layout() { return llvm::StructType::get(context, {ir.getInt64Ty(), ir.getPtrTy()}); } + llvm::StructType* list_layout() { return llvm::StructType::get(context, {ir.getInt64Ty(), ir.getPtrTy(), ir.getInt64Ty()}); } llvm::AllocaInst* entry_alloca(llvm::Type* type) { auto& entry = ir.GetInsertBlock()->getParent()->getEntryBlock(); #if LLVM_VERSION_MAJOR >= 24 @@ -303,6 +303,12 @@ class Generator { fail(callee.location, "method '" + callee.value + "' is private"); args.push_back(object.value); } else if (callee.kind == Expr::Kind::Name) { + if (callee.value == "len") { + if (expr.children.size() != 2) fail(expr.location, "len requires one argument"); + auto collection = expression(*expr.children[1]); + if (!is_list(collection.type)) fail(expr.location, "len requires a list or vector"); + return {ir.CreateLoad(ir.getInt64Ty(), ir.CreateStructGEP(list_layout(), collection.value, 0)), "number"}; + } if (callee.value == "input") return builtin_input(expr); if (callee.value == "read_file" || callee.value == "write_file" || callee.value == "append_file") return builtin_file(expr, callee.value); if (callee.value == "number" || callee.value == "decimal") return builtin_number(expr, callee.value); @@ -402,10 +408,8 @@ class Generator { if (lhs.type == "decimal") return ir.CreateFCmpOEQ(lhs.value, rhs.value); return ir.CreateICmpEQ(lhs.value, rhs.value); } - Value index_address(const Expr& e) { - auto list = expression(*e.children[0]); + Value index_address(const Expr& e, Value list, Value index) { if (!is_list(list.type)) fail(e.location, "indexing requires a list or vector"); - auto index = expression(*e.children[1]); require_type(index, "number", e.children[1]->location); auto layout = list_layout(); auto length = ir.CreateLoad(ir.getInt64Ty(), ir.CreateStructGEP(layout, list.value, 0)); @@ -449,17 +453,68 @@ class Generator { auto header = allocate(header_size); ir.CreateStore(ir.getInt64(values.size()), ir.CreateStructGEP(layout, header, 0)); ir.CreateStore(data, ir.CreateStructGEP(layout, header, 1)); + ir.CreateStore(ir.getInt64(values.empty() ? 1 : values.size()), ir.CreateStructGEP(layout, header, 2)); return {header, "list of " + element}; } if (e.kind == K::Index) { - auto address = index_address(e); + auto list = expression(*e.children[0]); + auto index = expression(*e.children[1]); + auto address = index_address(e, list, index); return {ir.CreateLoad(llvm_type(address.type, e.location), address.value), address.type}; } if (e.kind == K::Name) return lookup(e.value, e.location); if (e.kind == K::Member) return field(expression(*e.children[0]), e.value, e.location); if (e.kind == K::Call) return call(e); + auto check_bitwise_range = [&](llvm::Value* value) { + auto low = ir.CreateICmpSLT(value, ir.getInt64(-9007199254740991LL)); + auto high = ir.CreateICmpSGT(value, ir.getInt64(9007199254740991LL)); + raise_if(ir.CreateOr(low, high), "bitwise value exceeds exact numeric range"); + }; + if (e.kind == K::Unary) { + auto operand = expression(*e.children[0]); + auto op = e.value; + for (auto& c : op) c = static_cast(std::tolower(static_cast(c))); + if (op == "not" || op == "!") { + require_type(operand, "bool", e.location); + return {ir.CreateNot(operand.value), "bool"}; + } + if (op == "~") { + require_type(operand, "number", e.location); + check_bitwise_range(operand.value); + auto result = ir.CreateNot(operand.value); + check_bitwise_range(result); + return {result, "number"}; + } + if (op == "+" || op == "-") { + if (operand.type == "number") return {op == "-" ? ir.CreateNeg(operand.value) : operand.value, "number"}; + if (operand.type == "decimal") return {op == "-" ? ir.CreateFNeg(operand.value) : operand.value, "decimal"}; + } + fail(e.location, "invalid unary operator or operand"); + } if (e.kind == K::Binary) { auto lhs = expression(*e.children[0]); + auto op = e.value; + for (auto& c : op) c = static_cast(std::tolower(static_cast(c))); + if (op == "and" || op == "&&" || op == "or" || op == "||") { + require_type(lhs, "bool", e.location); + bool is_and = op == "and" || op == "&&"; + auto fn = ir.GetInsertBlock()->getParent(); + auto from = ir.GetInsertBlock(); + auto rhs_block = llvm::BasicBlock::Create(context, "logic.rhs", fn); + auto merge = llvm::BasicBlock::Create(context, "logic.end", fn); + if (is_and) ir.CreateCondBr(lhs.value, rhs_block, merge); + else ir.CreateCondBr(lhs.value, merge, rhs_block); + ir.SetInsertPoint(rhs_block); + auto rhs = expression(*e.children[1]); + require_type(rhs, "bool", e.children[1]->location); + auto rhs_end = ir.GetInsertBlock(); + ir.CreateBr(merge); + ir.SetInsertPoint(merge); + auto result = ir.CreatePHI(ir.getInt1Ty(), 2); + result->addIncoming(ir.getInt1(!is_and), from); + result->addIncoming(rhs.value, rhs_end); + return {result, "bool"}; + } auto rhs = expression(*e.children[1]); require_type(rhs, lhs.type, e.children[1]->location); if (e.value == "=" || e.value == "==" || e.value == "!=") { @@ -467,6 +522,24 @@ class Generator { return {e.value == "!=" ? ir.CreateNot(equal) : equal, "bool"}; } if (lhs.type == "number") { + if (e.value == "&" || e.value == "|" || e.value == "^" || e.value == "<<" || e.value == ">>") { + check_bitwise_range(lhs.value); + check_bitwise_range(rhs.value); + llvm::Value* result = nullptr; + if (e.value == "&") result = ir.CreateAnd(lhs.value, rhs.value); + else if (e.value == "|") result = ir.CreateOr(lhs.value, rhs.value); + else if (e.value == "^") result = ir.CreateXor(lhs.value, rhs.value); + else { + auto bad = ir.CreateOr(ir.CreateICmpSLT(rhs.value, ir.getInt64(0)), ir.CreateICmpSGE(rhs.value, ir.getInt64(64))); + raise_if(bad, "shift count must be between 0 and 63"); + if (e.value == "<<") { + result = ir.CreateShl(lhs.value, rhs.value); + raise_if(ir.CreateICmpNE(ir.CreateAShr(result, rhs.value), lhs.value), "bitwise shift overflow"); + } else result = ir.CreateAShr(lhs.value, rhs.value); + } + check_bitwise_range(result); + return {result, "number"}; + } if (e.value == "+") return {ir.CreateAdd(lhs.value, rhs.value), "number"}; if (e.value == "-") return {ir.CreateSub(lhs.value, rhs.value), "number"}; if (e.value == "*") return {ir.CreateMul(lhs.value, rhs.value), "number"}; @@ -533,6 +606,7 @@ class Generator { fail(stmt.location, "duplicate callable '" + stmt.name + "'"); if (stmt.foreign_symbol == "main" || stmt.foreign_symbol == "malloc" || stmt.foreign_symbol == "strcmp" || stmt.foreign_symbol == "abort" || + stmt.foreign_symbol == "realloc" || stmt.foreign_symbol == "memmove" || module.getNamedValue(stmt.foreign_symbol)) fail(stmt.location, "duplicate or reserved external symbol '" + stmt.foreign_symbol + "'"); std::vector args; @@ -713,9 +787,14 @@ class Generator { } case Stmt::Kind::Set: { if (stmt.target) { - auto address = index_address(*stmt.target); - auto value = expression(*stmt.expression, address.type); - require_type(value, address.type, stmt.expression->location); + const auto& target = *stmt.target; + auto list = expression(*target.children[0]); + if (!is_list(list.type)) fail(target.location, "indexing requires a list or vector"); + auto index = expression(*target.children[1]); + require_type(index, "number", target.children[1]->location); + auto value = expression(*stmt.expression, list.type.substr(8)); + require_type(value, list.type.substr(8), stmt.expression->location); + auto address = index_address(target, list, index); ir.CreateStore(value.value, address.value); break; } @@ -728,6 +807,72 @@ class Generator { ir.CreateStore(value.value, it->second.value); break; } + case Stmt::Kind::Add: + case Stmt::Kind::Remove: { + auto value = expression(*stmt.expression); + auto list = expression(*stmt.target); + if (!is_list(list.type)) fail(stmt.target->location, "add and remove require a list or vector"); + auto type = list.type.substr(8); + require_type(value, type, stmt.expression->location); + auto layout = list_layout(); + auto length_ptr = ir.CreateStructGEP(layout, list.value, 0); + auto data_ptr = ir.CreateStructGEP(layout, list.value, 1); + auto capacity_ptr = ir.CreateStructGEP(layout, list.value, 2); + auto length = ir.CreateLoad(ir.getInt64Ty(), length_ptr); + auto element_type = llvm_type(type, stmt.location); + auto null = llvm::ConstantPointerNull::get(ir.getPtrTy()); + auto element_size = ir.CreatePtrToInt(ir.CreateGEP(element_type, null, ir.getInt64(1)), ir.getInt64Ty()); + auto fn = ir.GetInsertBlock()->getParent(); + if (stmt.kind == Stmt::Kind::Add) { + auto capacity = ir.CreateLoad(ir.getInt64Ty(), capacity_ptr); + auto grow = llvm::BasicBlock::Create(context, "list.grow", fn); + auto ready = llvm::BasicBlock::Create(context, "list.ready", fn); + ir.CreateCondBr(ir.CreateICmpEQ(length, capacity), grow, ready); + ir.SetInsertPoint(grow); + auto max_capacity = ir.CreateUDiv(ir.getInt64(INT64_MAX), element_size); + raise_if(ir.CreateICmpUGT(capacity, ir.CreateUDiv(max_capacity, ir.getInt64(2))), "list too large"); + auto new_capacity = ir.CreateMul(capacity, ir.getInt64(2)); + auto data = ir.CreateLoad(ir.getPtrTy(), data_ptr); + auto signature = llvm::FunctionType::get(ir.getPtrTy(), {ir.getPtrTy(), ir.getInt64Ty()}, false); + auto resized = ir.CreateCall(module.getOrInsertFunction("realloc", signature), {data, ir.CreateMul(new_capacity, element_size)}); + raise_if(ir.CreateIsNull(resized), "cannot grow list"); + ir.CreateStore(resized, data_ptr); + ir.CreateStore(new_capacity, capacity_ptr); + ir.CreateBr(ready); + ir.SetInsertPoint(ready); + auto current = ir.CreateLoad(ir.getPtrTy(), data_ptr); + ir.CreateStore(value.value, ir.CreateGEP(element_type, current, length)); + ir.CreateStore(ir.CreateAdd(length, ir.getInt64(1)), length_ptr); + } else { + auto data = ir.CreateLoad(ir.getPtrTy(), data_ptr); + auto start = ir.GetInsertBlock(); + auto search = llvm::BasicBlock::Create(context, "list.search", fn); + auto compare = llvm::BasicBlock::Create(context, "list.compare", fn); + auto next = llvm::BasicBlock::Create(context, "list.next", fn); + auto found = llvm::BasicBlock::Create(context, "list.found", fn); + ir.CreateBr(search); + ir.SetInsertPoint(search); + auto index = ir.CreatePHI(ir.getInt64Ty(), 2); + index->addIncoming(ir.getInt64(0), start); + raise_if(ir.CreateICmpUGE(index, length), "item not found in list"); + ir.CreateBr(compare); + ir.SetInsertPoint(compare); + auto item = ir.CreateLoad(element_type, ir.CreateGEP(element_type, data, index)); + ir.CreateCondBr(equal_values({item, type}, value, stmt.location), found, next); + ir.SetInsertPoint(next); + auto increment = ir.CreateAdd(index, ir.getInt64(1)); + ir.CreateBr(search); + index->addIncoming(increment, next); + ir.SetInsertPoint(found); + auto after = ir.CreateAdd(index, ir.getInt64(1)); + auto dest = ir.CreateGEP(element_type, data, index); + auto src = ir.CreateGEP(element_type, data, after); + auto bytes = ir.CreateMul(ir.CreateSub(length, after), element_size); + ir.CreateMemMove(dest, llvm::MaybeAlign(), src, llvm::MaybeAlign(), bytes); + ir.CreateStore(ir.CreateSub(length, ir.getInt64(1)), length_ptr); + } + break; + } case Stmt::Kind::Repeat: { auto count = expression(*stmt.expression); require_type(count, "number", stmt.expression->location); @@ -790,8 +935,6 @@ class Generator { auto list = expression(*stmt.expression); if (!is_list(list.type)) fail(stmt.expression->location, "for each requires a list or vector"); auto layout = list_layout(); - auto length = ir.CreateLoad(ir.getInt64Ty(), ir.CreateStructGEP(layout, list.value, 0)); - auto data = ir.CreateLoad(ir.getPtrTy(), ir.CreateStructGEP(layout, list.value, 1)); auto fn = ir.GetInsertBlock()->getParent(); auto preheader = ir.GetInsertBlock(); auto condition = llvm::BasicBlock::Create(context, "for.condition", fn); @@ -802,8 +945,10 @@ class Generator { ir.SetInsertPoint(condition); auto index = ir.CreatePHI(ir.getInt64Ty(), 2); index->addIncoming(ir.getInt64(0), preheader); + auto length = ir.CreateLoad(ir.getInt64Ty(), ir.CreateStructGEP(layout, list.value, 0)); ir.CreateCondBr(ir.CreateICmpULT(index, length), body, done); ir.SetInsertPoint(body); + auto data = ir.CreateLoad(ir.getPtrTy(), ir.CreateStructGEP(layout, list.value, 1)); auto type = list.type.substr(8); auto element_type = llvm_type(type, stmt.location); auto value = ir.CreateLoad(element_type, ir.CreateGEP(element_type, data, index)); diff --git a/TellC/src/parser.cpp b/TellC/src/parser.cpp index 272d93f..a084836 100644 --- a/TellC/src/parser.cpp +++ b/TellC/src/parser.cpp @@ -19,7 +19,7 @@ std::vector lex(const std::string& s) { if(std::isalpha(c)||c=='_'||c>=128) { while(i=128)) break; ++i; ++col; } add(s.substr(start,i-start),l,Token::Word); continue; } if(std::isdigit(c)) { while(i'&&s[i]=='=')||(c=='*'&&s[i]=='*')||(c=='/'&&s[i]=='/'))) {op+=s[i++];++col;} add(op,l,Token::Symbol); + ++i; ++col; std::string op(1,c); if(i'&&s[i]=='=')||(c=='*'&&s[i]=='*')||(c=='/'&&s[i]=='/')||(c=='&'&&s[i]=='&')||(c=='|'&&s[i]=='|')||(c=='<'&&s[i]=='<')||(c=='>'&&s[i]=='>'))) {op+=s[i++];++col;} add(op,l,Token::Symbol); } v.push_back({"",{line,col},Token::End}); return v; } @@ -45,7 +45,7 @@ class Parser { void separators(){while(take("\n")||take(";")) {}} void end(){if(peek().kind==Token::End) return; if(!is("\n")&&!is(";")) throw Error(peek().loc,"expected newline or ';' after statement"); separators();} std::unique_ptr node(Expr::Kind k, Token a) { auto e=std::make_unique(); e->kind=k;e->location=a.loc;e->value=a.text;return e; } - int prec(const std::string& s){if(s=="or")return 1;if(s=="and")return 2;if(s=="="||s=="=="||s=="!="||s=="<"||s==">"||s=="<="||s==">="||s=="in")return 3;if(s=="+"||s=="-"||s=="plus"||s=="minus")return 4;if(s=="*"||s=="/"||s=="//"||s=="%"||s=="times")return 5;if(s=="**")return 6;return 0;} + int prec(const std::string& s){if(s=="or"||s=="||")return 1;if(s=="and"||s=="&&")return 2;if(s=="is"||s=="equal"||s=="="||s=="=="||s=="!="||s=="<"||s==">"||s=="<="||s==">="||s=="in")return 3;if(s=="|")return 4;if(s=="^")return 5;if(s=="&")return 6;if(s=="<<"||s==">>")return 7;if(s=="+"||s=="-"||s=="plus"||s=="minus")return 8;if(s=="*"||s=="/"||s=="//"||s=="%"||s=="times")return 9;if(s=="**")return 11;return 0;} std::unique_ptr expr(int min=1){ Token a=get(); std::unique_ptr x; auto w=lower(a.text); @@ -53,7 +53,7 @@ class Parser { else if(a.kind==Token::String) x=node(Expr::Kind::String,a); else if(w=="true"||w=="false") x=node(Expr::Kind::Boolean,a); else if(w=="nothing"||w=="null"||w=="nil") x=node(Expr::Kind::Nothing,a); - else if(w=="not"||w=="-"||w=="+") {x=node(Expr::Kind::Unary,a);x->children.push_back(expr(6));} + else if(w=="not"||w=="!"||w=="~"||w=="-"||w=="+") {x=node(Expr::Kind::Unary,a);x->children.push_back(expr(10));} else if(a.text=="("){x=expr();expect(")");} else if(a.text=="["){x=node(Expr::Kind::List,a);if(!take("]")){do{x->children.push_back(expr());}while(take(","));expect("]");}} else if(a.kind==Token::Word) { a.text=call_name(a.text); x=node(Expr::Kind::Name,a); } @@ -63,7 +63,12 @@ class Parser { if(is("[")){Token op=get();auto n=node(Expr::Kind::Index,op);n->children.push_back(std::move(x));n->children.push_back(expr());expect("]");x=std::move(n);continue;} if(is(".") && p+1children.push_back(std::move(x));x=std::move(n);continue;} if(is("times") && p+1children.push_back(std::move(x));n->children.push_back(expr(bp+(op.text=="**"?0:1)));x=std::move(n); + int bp=prec(lower(peek().text));if(bpchildren.push_back(std::move(x));n->children.push_back(expr(bp+(op.text=="**"?0:1)));x=std::move(n); } return x; } std::vector body(const std::string& until, bool branches=false){std::vector out;separators();while(peek().kind!=Token::End&&!is("end")&&!(branches&&(is("otherwise")||is("else")))) out.push_back(statement()); if(peek().kind==Token::End) throw Error(peek().loc,"expected end "+until); return out;} @@ -87,6 +92,7 @@ class Parser { if(take("set")){s.kind=Stmt::Kind::Set; Token target=peek(); s.name=name(); if(is("[")){s.target=node(Expr::Kind::Name,target);while(is("[")){Token op=get();auto index=node(Expr::Kind::Index,op);index->children.push_back(std::move(s.target));index->children.push_back(expr());expect("]");s.target=std::move(index);}} expect("to");s.expression=expr();end();return s;} + if(take("add") || take("remove")){s.kind=lower(first.text)=="add"?Stmt::Kind::Add:Stmt::Kind::Remove;s.expression=expr();expect(s.kind==Stmt::Kind::Add?"to":"from");s.target=expr();end();return s;} if(take("say")||take("show")||take("print")){s.kind=Stmt::Kind::Say;s.expression=expr();end();return s;} if(take("return")||take("give")){s.kind=Stmt::Kind::Return;if(lower(first.text)=="give")expect("back");if(!is("\n")&&!is(";")&&peek().kind!=Token::End)s.expression=expr();end();return s;} if(take("raise")){expect("an");expect("error");expect("saying");s.kind=Stmt::Kind::Raise;s.expression=expr();end();return s;} diff --git a/TellC/tests/compiler_integration.cmake b/TellC/tests/compiler_integration.cmake index 1af6927..8bec51a 100644 --- a/TellC/tests/compiler_integration.cmake +++ b/TellC/tests/compiler_integration.cmake @@ -105,3 +105,7 @@ execute_process(COMMAND "${TELL_WORK_DIR}/uncaught_error" RESULT_VARIABLE uncaug if(NOT uncaught_result EQUAL 1 OR NOT uncaught_output STREQUAL "" OR NOT uncaught_error STREQUAL "Error: invalid number\n") message(FATAL_ERROR "Unexpected uncaught error: ${uncaught_result}, [${uncaught_output}], [${uncaught_error}]") endif() +compile_and_run(mutable_collections tests/fixtures/mutable_collections.tell "5\n7\n9\n5\n2\n7\n2\n3\n4\n9\n8\ntwo\nitem not found in list\n2\n1\n3\n2\nfalse\n") +expect_compile_failure(mutable_type tests/fixtures/mutable_type.tell "expected 'number', got 'text'") +expect_compile_failure(mutable_target tests/fixtures/mutable_target.tell "add and remove require a list or vector") +compile_and_run(operators tests/fixtures/operators.tell "2\n7\n5\n-6\n16\n-2\n13\ntrue\nfalse\ntrue\ntrue\nfalse\ntrue\ninvalid shift\nshift out of range\n") diff --git a/TellC/tests/fixtures/mutable_collections.tell b/TellC/tests/fixtures/mutable_collections.tell new file mode 100644 index 0000000..16937c5 --- /dev/null +++ b/TellC/tests/fixtures/mutable_collections.tell @@ -0,0 +1,48 @@ +to append_value(list of number: items) out void: + add 4 to items +end function +to change(list of number: items) out number: + add 9 to items + return 7 +end function +let vector of number values be [] +let list of number alias be values +add 1 to values +add 2 to values +add 2 to alias +add 3 to values +append_value(alias) +say len(values) +set values[0] to change(alias) +say values[0] +say values[5] +remove 2 from alias +say len(values) +say values[1] +for each item in values: + say item + if item = 4: + add 8 to alias + end if +end for +let list of text words be ["one", "two", "two"] +remove "two" from words +say words[1] +try: + remove "missing" from words +if something goes wrong: + say error_message +end try +say len(words) +let list of number shrinking be [1, 2, 3] +for each current in shrinking: + say current + if current = 1: + remove 1 from shrinking + end if +end for +say len(shrinking) +let list of bool flags be [true] +add false to flags +remove true from flags +say flags[0] diff --git a/TellC/tests/fixtures/mutable_target.tell b/TellC/tests/fixtures/mutable_target.tell new file mode 100644 index 0000000..12d3078 --- /dev/null +++ b/TellC/tests/fixtures/mutable_target.tell @@ -0,0 +1 @@ +add 1 to 3 diff --git a/TellC/tests/fixtures/mutable_type.tell b/TellC/tests/fixtures/mutable_type.tell new file mode 100644 index 0000000..5daab01 --- /dev/null +++ b/TellC/tests/fixtures/mutable_type.tell @@ -0,0 +1,2 @@ +let list of number items be [] +add "wrong" to items From 32224753f252e80ff709a9f69b1cfb0f3e39065f Mon Sep 17 00:00:00 2001 From: Neonwave175 Date: Wed, 18 Nov 2026 22:20:41 +0530 Subject: [PATCH 2/3] Build and test a standalone TellC language server --- .github/workflows/tellc.yml | 11 +- TellC/CMakeLists.txt | 24 ++++- TellC/src/lsp.cpp | 199 +++++++++++++++++++++++++++++++++++ docs/IDE.md | 2 + lsp/README.md | 36 +++---- lsp/pyproject.toml | 16 --- lsp/tell_lsp/__init__.py | 1 - lsp/tell_lsp/__main__.py | 144 ------------------------- lsp/test_lsp.py | 85 ++++++++------- scripts/build-tellc-macos.sh | 1 + 10 files changed, 290 insertions(+), 229 deletions(-) create mode 100644 TellC/src/lsp.cpp delete mode 100644 lsp/pyproject.toml delete mode 100644 lsp/tell_lsp/__init__.py delete mode 100644 lsp/tell_lsp/__main__.py diff --git a/.github/workflows/tellc.yml b/.github/workflows/tellc.yml index 12fdc7d..589d53b 100644 --- a/.github/workflows/tellc.yml +++ b/.github/workflows/tellc.yml @@ -1,8 +1,11 @@ name: TellC LLVM compiler on: - # The embedded LLVM/LLD build is intentionally excluded from normal PRs: - # upstream builds are expensive and the native Go/runtime suites run on PRs. + pull_request: + paths: + - "TellC/**" + - "lsp/**" + - ".github/workflows/tellc.yml" schedule: - cron: "17 2 * * 1" workflow_dispatch: @@ -39,10 +42,11 @@ jobs: - name: Build TellC run: cmake --build build/tellc --parallel 2 - - name: Run parser, CLI, and compiler integration tests + - name: Run compiler and language server tests run: ctest --test-dir build/tellc --output-on-failure macos-standalone: + if: github.event_name != 'pull_request' name: Apple silicon / embedded LLVM + LLD runs-on: macos-15 steps: @@ -70,6 +74,7 @@ jobs: if-no-files-found: error macos-intel: + if: github.event_name != 'pull_request' name: Intel macOS / external LLVM toolchain runs-on: macos-15-intel steps: diff --git a/TellC/CMakeLists.txt b/TellC/CMakeLists.txt index b9dcf3e..be4ce5e 100644 --- a/TellC/CMakeLists.txt +++ b/TellC/CMakeLists.txt @@ -58,7 +58,13 @@ add_library(tell_parser src/parser.cpp) target_include_directories(tell_parser PUBLIC include) target_compile_features(tell_parser PUBLIC cxx_std_17) -add_executable(tellc src/main.cpp src/codegen.cpp) +add_library(tell_codegen STATIC src/codegen.cpp) +target_include_directories(tell_codegen SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) +target_compile_definitions(tell_codegen PRIVATE ${LLVM_DEFINITIONS}) +target_link_libraries(tell_codegen PUBLIC tell_parser PRIVATE ${TELL_LLVM_LIBS}) + +add_executable(tellc src/main.cpp) +target_link_libraries(tellc PRIVATE tell_codegen) target_compile_features(tellc PRIVATE cxx_std_17) target_include_directories(tellc SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) target_compile_definitions(tellc PRIVATE ${LLVM_DEFINITIONS} TELL_LLC_PATH="${TELL_LLC_PATH}") @@ -80,10 +86,11 @@ if(TELL_STANDALONE_LINUX) target_include_directories(tellc SYSTEM PRIVATE ${LLD_INCLUDE_DIRS}) target_include_directories(tellc PRIVATE "${runtime_dir}") target_link_options(tellc PRIVATE -s) - target_link_libraries(tellc PRIVATE tell_parser ${TELL_LLVM_LIBS}) + target_compile_definitions(tell_codegen PRIVATE TELL_STANDALONE_LINUX TELL_EMBEDDED_CODEGEN) elseif(TELL_STANDALONE_MACOS) target_compile_definitions(tellc PRIVATE TELL_STANDALONE_MACOS TELL_EMBEDDED_CODEGEN) target_include_directories(tellc SYSTEM PRIVATE "${LLD_SOURCE_DIR}/include") + target_compile_definitions(tell_codegen PRIVATE TELL_EMBEDDED_CODEGEN) # Keep an escape hatch for custom LLVM builds that still expose shared zstd. set(TELL_ZSTD_STATIC_LIBRARY "" CACHE FILEPATH "Static zstd library used by LLVM") @@ -96,17 +103,24 @@ elseif(TELL_STANDALONE_MACOS) endif() target_link_libraries(tellc PRIVATE - tell_parser "${LLD_BUILD_DIR}/lib/liblldMachO.a" "${LLD_BUILD_DIR}/lib/liblldCommon.a" ${TELL_LLVM_LIBS}) -else() - target_link_libraries(tellc PRIVATE tell_parser ${TELL_LLVM_LIBS}) endif() +add_executable(tell-lsp src/lsp.cpp) +target_include_directories(tell-lsp SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) +target_compile_definitions(tell-lsp PRIVATE ${LLVM_DEFINITIONS}) +target_link_libraries(tell-lsp PRIVATE tell_codegen) +install(TARGETS tell-lsp RUNTIME DESTINATION bin) + include(CTest) if(BUILD_TESTING) find_package(Python3 REQUIRED COMPONENTS Interpreter) + add_test(NAME language_server + COMMAND ${CMAKE_COMMAND} -E env "TELL_LSP_BINARY=$" + "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/../lsp/test_lsp.py") + add_test(NAME cli_smoke COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/cli_smoke.py" diff --git a/TellC/src/lsp.cpp b/TellC/src/lsp.cpp new file mode 100644 index 0000000..d707abd --- /dev/null +++ b/TellC/src/lsp.cpp @@ -0,0 +1,199 @@ +#include "codegen.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { +using llvm::StringRef; +namespace json = llvm::json; + +constexpr StringRef keywords[] = { + "let", "let number", "let decimal", "let text", "let bool", "object", + "own", "others", "end object", "to", "out", "end function", "self", "if", + "otherwise", "end if", "while", "end while", "return", "give back", + "say", "true", "false", "extern cpp function", "extern c function", +}; + +struct DiagnosticRange { + size_t line = 0; + size_t start = 0; + size_t end = 0; +}; + +DiagnosticRange diagnostic_range(StringRef source, tell::Location location) { + DiagnosticRange range; + auto parts = source.split('\n'); + while (range.line + 1 < location.line && parts.first.size() < source.size()) { + source = parts.second; + ++range.line; + parts = source.split('\n'); + } + + StringRef line = parts.first.rtrim('\r'); + const size_t target = std::min(location.column - 1, line.size()); + size_t offset = 0; + while (offset < line.size()) { + const unsigned bytes = llvm::getNumBytesForUTF8(line[offset]); + if (offset + bytes > target) { + range.end = range.start + (bytes == 4 ? 2 : 1); + return range; + } + offset += bytes; + range.start += bytes == 4 ? 2 : 1; + } + range.end = range.start; + return range; +} + +json::Object diagnostic(StringRef message, DiagnosticRange range = {}) { + json::Object positions{ + {"start", json::Object{{"line", range.line}, {"character", range.start}}}, + {"end", json::Object{{"line", range.line}, {"character", range.end}}}, + }; + return json::Object{{"range", std::move(positions)}, + {"severity", 1}, {"source", "tellc"}, + {"message", json::isUTF8(message) ? message.str() : json::fixUTF8(message)}}; +} + +json::Array diagnostics(StringRef source) { + try { + tell::check_program(tell::parse(source.str())); + return {}; + } catch (const tell::Error& error) { + return json::Array{diagnostic(error.what(), diagnostic_range(source, error.location))}; + } catch (const std::exception& error) { + return json::Array{diagnostic(std::string("tellc: ") + error.what())}; + } +} + +class Server { + std::map documents; + bool shutdown_requested = false; + + static void send(json::Object message) { + std::string body; + llvm::raw_string_ostream(body) << json::Value(std::move(message)); + std::cout << "Content-Length: " << body.size() << "\r\n\r\n" << body << std::flush; + } + + static void publish(StringRef uri, json::Array items, + std::optional version = std::nullopt) { + json::Object params{{"uri", uri}, {"diagnostics", std::move(items)}}; + if (version) params["version"] = *version; + send(json::Object{{"jsonrpc", "2.0"}, {"method", "textDocument/publishDiagnostics"}, + {"params", std::move(params)}}); + } + + void update_document(StringRef method, const json::Object* params) { + const auto* document = params ? params->getObject("textDocument") : nullptr; + if (!document) return; + auto uri = document->getString("uri"); + if (!uri) return; + + if (method == "textDocument/didClose") { + documents.erase(uri->str()); + publish(*uri, {}); + return; + } + + std::optional source; + if (method == "textDocument/didOpen") { + source = document->getString("text"); + } else { + if (documents.find(uri->str()) == documents.end()) return; + const auto* changes = params->getArray("contentChanges"); + if (!changes || changes->empty()) return; + const auto* change = changes->back().getAsObject(); + if (change && !change->get("range")) source = change->getString("text"); + } + if (!source) return; + + auto& buffer = documents[uri->str()]; + buffer = source->str(); + publish(*uri, diagnostics(buffer), document->getInteger("version")); + } + + bool handle(const json::Object& message) { + auto method = message.getString("method"); + if (!method) return true; + if (*method == "exit") return false; + + const auto* id = message.get("id"); + json::Value result = nullptr; + if (*method == "initialize") { + result = json::Object{ + {"capabilities", json::Object{ + {"textDocumentSync", json::Object{{"openClose", true}, {"change", 1}}}, + {"completionProvider", json::Object{}}, + }}, + {"serverInfo", json::Object{{"name", "tell-lsp"}, {"version", "0.1.0"}}}, + }; + } else if (*method == "shutdown") { + shutdown_requested = true; + } else if (*method == "textDocument/didOpen" || *method == "textDocument/didChange" || + *method == "textDocument/didClose") { + update_document(*method, message.getObject("params")); + } else if (*method == "textDocument/completion") { + json::Array items; + for (StringRef word : keywords) items.push_back(json::Object{{"label", word}, {"kind", 14}}); + result = std::move(items); + } else if (id) { + send(json::Object{{"jsonrpc", "2.0"}, {"id", *id}, + {"error", json::Object{{"code", -32601}, {"message", "Method not found"}}}}); + return true; + } + if (id) send(json::Object{{"jsonrpc", "2.0"}, {"id", *id}, {"result", std::move(result)}}); + return true; + } + +public: + int run() { + // Allow large editor buffers while bounding the allocation from a frame header. + constexpr size_t max_message_size = 32 * 1024 * 1024; + for (;;) { + std::optional length; + for (std::string header;;) { + if (!std::getline(std::cin, header)) + return shutdown_requested && header.empty() && !length ? 0 : 1; + auto line = StringRef(header).rtrim('\r'); + if (line.empty()) break; + auto separator = line.find(':'); + if (separator == StringRef::npos) return 1; + auto fields = line.split(':'); + if (fields.first.trim().equals_insensitive("Content-Length")) { + size_t size; + if (length || fields.second.trim().getAsInteger(10, size) || size > max_message_size) + return 1; + length = size; + } + } + if (!length) return 1; + std::string body(*length, '\0'); + if (!std::cin.read(body.data(), body.size())) return 1; + auto value = json::parse(body); + if (!value) { + llvm::consumeError(value.takeError()); + return 1; + } + const auto* message = value->getAsObject(); + if (!message) return 1; + if (!handle(*message)) return shutdown_requested ? 0 : 1; + } + } +}; +} // namespace + +int main() { + std::cin.tie(nullptr); + return Server{}.run(); +} diff --git a/docs/IDE.md b/docs/IDE.md index 3a4b4ec..4cf795d 100644 --- a/docs/IDE.md +++ b/docs/IDE.md @@ -44,3 +44,5 @@ editor reach sufficient parity. The Python runtime should receive only security, compatibility, and migration fixes; new runtime features belong in Go. The IDE should receive only critical compatibility fixes; new editor features belong in GPUI Studio. + +For editor diagnostics for the typed TellC dialect, see the separate [TellC language server](../lsp/README.md). diff --git a/lsp/README.md b/lsp/README.md index 0f43a50..2716e91 100644 --- a/lsp/README.md +++ b/lsp/README.md @@ -1,22 +1,19 @@ # TellC language server -The language server is a separate Python package for editors that support LSP. -It does not import or require the Tell Python runtime or Tk IDE. It requires -Python 3.10+ and a TellC compiler built from this checkout; rebuild `tellc` -after updating it so that `--check-stdin` is available. See -[TellC build instructions](../TellC/README.md). +The TellC language server is a C++ executable that speaks LSP over stdio. +It checks unsaved buffers with the same parser and LLVM checks as TellC. -Install this package independently: +Build the server from a configured TellC build directory. See the +[TellC build guide](../TellC/docs/BUILDING.md) for LLVM setup. ```sh -python3 -m pip install ./lsp +cmake --build build/tellc --target tell-lsp +cmake --install build/tellc --prefix "$HOME/.local" ``` -Configure your editor to run `tell-lsp` over stdio for `.tell` files. The -server runs `tellc --check-stdin` on the unsaved buffer on each change. Put the -compiler on the editor's PATH or set `TELL_LSP_COMPILER` to its executable -path. An unavailable or incompatible checker is shown as an editor diagnostic. -The server never executes the program or writes a compiled output file. +The Apple silicon build script also builds the server at +`build/tellc-macos-arm64/tellc/tell-lsp`. If your editor still points to +the old Python launcher, set its command to the new executable. For Neovim 0.11 or newer, run the setup script from the repository root. It installs file detection, syntax highlighting, diagnostics, and automatic @@ -31,18 +28,9 @@ on your PATH. You can also pass the executable path as the first argument. Restart Neovim and open a `.tell` file. The script leaves your `init.lua` in place and refuses to overwrite existing Tell files it did not install. -From a checkout without installing, run `python3 -m tell_lsp` with the `lsp` -directory on `PYTHONPATH`. - -Diagnostics use the TellC compiler's parser and code generation checks, so -they support its typed functions, objects and access rules, and report language -features the current compiler cannot generate. The first compiler error is -reported per buffer. The server offers keyword completions; hover, navigation, -cross-file analysis, and incremental synchronization are not supported. It is -intended for TellC's typed dialect, not the older Python or Go runtime dialects. - -Run the integration tests with a freshly built compiler: +The server reports the first TellC error per buffer and offers keyword +completion. It uses full document synchronization. ```sh -TELL_LSP_COMPILER=/absolute/path/to/tellc python3 -m unittest discover -s lsp -v +ctest --test-dir build/tellc -R '^language_server$' --output-on-failure ``` diff --git a/lsp/pyproject.toml b/lsp/pyproject.toml deleted file mode 100644 index 8f7c57f..0000000 --- a/lsp/pyproject.toml +++ /dev/null @@ -1,16 +0,0 @@ -[build-system] -requires = ["setuptools>=68"] -build-backend = "setuptools.build_meta" - -[project] -name = "tellc-language-server" -version = "0.1.0" -description = "Language server for the typed TellC language" -requires-python = ">=3.10" -license = { text = "MIT" } - -[project.scripts] -tell-lsp = "tell_lsp.__main__:main" - -[tool.setuptools] -packages = ["tell_lsp"] diff --git a/lsp/tell_lsp/__init__.py b/lsp/tell_lsp/__init__.py deleted file mode 100644 index 0958167..0000000 --- a/lsp/tell_lsp/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""TellC language server, independent of the Tell Python runtime and IDE.""" diff --git a/lsp/tell_lsp/__main__.py b/lsp/tell_lsp/__main__.py deleted file mode 100644 index b5fb76f..0000000 --- a/lsp/tell_lsp/__main__.py +++ /dev/null @@ -1,144 +0,0 @@ -"""A small, dependency-free Language Server Protocol server for Tell.""" -from __future__ import annotations - -import json -import os -import re -import subprocess -import sys -from typing import BinaryIO - -# TellC's typed language and object declarations. -COMPLETIONS = ( - "let", "let number", "let decimal", "let text", "let bool", - "object", "own", "others", "end object", "to", "out", "end function", - "self", "if", "otherwise", "end if", "while", "end while", "return", - "give back", "say", "true", "false", "extern cpp function", "extern c function", -) - - -def _utf16_length(text: str) -> int: - return len(text.encode("utf-16-le")) // 2 - - -def diagnostics(source: str, uri: str) -> list[dict]: - compiler = os.environ.get("TELL_LSP_COMPILER", "tellc") - try: - result = subprocess.run([compiler, "--check-stdin"], input=source.encode("utf-8"), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15) - except (OSError, subprocess.TimeoutExpired) as exc: - message = f"Could not run TellC checker ({compiler}): {exc}. Set TELL_LSP_COMPILER to a compatible tellc." - result = None - else: - if result.returncode == 0: - return [] - message = result.stderr.decode("utf-8", errors="replace").strip() or "TellC checker failed without a diagnostic" - line = column = 0 - match = re.match(r"^:(\d+):(\d+):\s*(.*)$", message, re.DOTALL) - lines = source.split("\n") - if match: - line = min(max(int(match.group(1)) - 1, 0), len(lines) - 1) - # TellC locations count UTF-8 bytes; LSP positions default to UTF-16. - text = lines[line].rstrip("\r") - byte_offset = max(int(match.group(2)) - 1, 0) - prefix = text.encode("utf-8")[:byte_offset].decode("utf-8", errors="ignore") - column = _utf16_length(prefix) - length = _utf16_length(text[len(prefix):len(prefix) + 1]) - message = match.group(3) - else: - length = 0 - return [{"range": {"start": {"line": line, "character": column}, - "end": {"line": line, "character": column + length}}, - "severity": 1, "source": "tellc", "message": message}] - - -class Server: - def __init__(self, reader: BinaryIO, writer: BinaryIO): - self.reader, self.writer = reader, writer - self.documents: dict[str, tuple[str, int | None]] = {} - self.shutdown_requested = False - - def send(self, message: dict) -> None: - data = json.dumps(message, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - self.writer.write(f"Content-Length: {len(data)}\r\n\r\n".encode("ascii") + data) - self.writer.flush() - - def publish(self, uri: str, source: str, version: int | None) -> None: - params = {"uri": uri, "diagnostics": diagnostics(source, uri)} - if version is not None: - params["version"] = version - self.send({"jsonrpc": "2.0", "method": "textDocument/publishDiagnostics", "params": params}) - - def handle(self, message: dict) -> bool: - method = message.get("method") - params = message.get("params") or {} - request = "id" in message - result = None - error = None - if method == "exit": - return False - if method == "initialize": - result = {"capabilities": {"textDocumentSync": {"openClose": True, "change": 1}, - "completionProvider": {}}, - "serverInfo": {"name": "tell-lsp", "version": "0.1.0"}} - elif method == "shutdown": - self.shutdown_requested = True - elif method == "textDocument/didOpen": - doc = params["textDocument"] - self.documents[doc["uri"]] = (doc["text"], doc.get("version")) - self.publish(doc["uri"], doc["text"], doc.get("version")) - elif method == "textDocument/didChange": - doc = params["textDocument"] - changes = params.get("contentChanges", []) - if doc["uri"] in self.documents and changes: - source = changes[-1]["text"] - self.documents[doc["uri"]] = (source, doc.get("version")) - self.publish(doc["uri"], source, doc.get("version")) - elif method == "textDocument/didClose": - uri = params["textDocument"]["uri"] - self.documents.pop(uri, None) - self.send({"jsonrpc": "2.0", "method": "textDocument/publishDiagnostics", - "params": {"uri": uri, "diagnostics": []}}) - elif method == "textDocument/completion": - result = [{"label": word, "kind": 14} for word in COMPLETIONS] - elif request: - error = {"code": -32601, "message": "Method not found"} - if request: - response = {"jsonrpc": "2.0", "id": message["id"]} - response["error" if error else "result"] = error if error else result - self.send(response) - return True - - def run(self) -> int: - while True: - headers = {} - while True: - line = self.reader.readline() - if not line: - return 0 if self.shutdown_requested else 1 - if line in (b"\r\n", b"\n"): - break - name, sep, value = line.partition(b":") - if not sep: - return 1 - headers[name.strip().lower()] = value.strip() - try: - length = int(headers[b"content-length"]) - if length < 0: - return 1 - data = self.reader.read(length) - if len(data) != length: - return 1 - message = json.loads(data.decode("utf-8")) - except (KeyError, ValueError, UnicodeError): - return 1 - if not self.handle(message): - return 0 if self.shutdown_requested else 1 - - -def main() -> int: - return Server(sys.stdin.buffer, sys.stdout.buffer).run() - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/lsp/test_lsp.py b/lsp/test_lsp.py index 573b77f..ccf44a9 100644 --- a/lsp/test_lsp.py +++ b/lsp/test_lsp.py @@ -2,14 +2,11 @@ import os from pathlib import Path import subprocess -import sys import unittest -from tell_lsp.__main__ import diagnostics - ROOT = Path(__file__).resolve().parent.parent -COMPILER = Path(os.environ.get("TELL_LSP_COMPILER", ROOT / "build/tellc-macos-arm64/tellc/tellc")) -os.environ["TELL_LSP_COMPILER"] = str(COMPILER) +BINARY = os.environ.get("TELL_LSP_BINARY", "tell-lsp") +URI = "file:///test.tell" def frame(message): @@ -27,51 +24,67 @@ def messages(data): return result -@unittest.skipUnless(COMPILER.is_file(), "set TELL_LSP_COMPILER to a built tellc") class LanguageServerTests(unittest.TestCase): + def exchange(self, requests): + lifecycle = [ + {"id": 1, "method": "initialize", "params": {}}, + {"method": "initialized", "params": {}}, + *requests, + {"id": 3, "method": "shutdown"}, + {"method": "exit"}, + ] + process = subprocess.run( + [BINARY], + input=b"".join(frame({"jsonrpc": "2.0", **request}) for request in lifecycle), + capture_output=True, + check=True, + timeout=15, + env={**os.environ, "TELL_LSP_COMPILER": "/nonexistent/tellc"}, + ) + self.assertEqual(process.stderr, b"") + return messages(process.stdout) + + def diagnostics(self, source): + responses = self.exchange([ + {"method": "textDocument/didOpen", "params": {"textDocument": {"uri": URI, "text": source}}}, + ]) + return responses[1]["params"]["diagnostics"] + def test_tellc_objects_and_type_errors(self): - self.assertEqual(diagnostics((ROOT / "TellC/examples/objects.tell").read_text(), "file:///test.tell"), []) + self.assertEqual(self.diagnostics((ROOT / "TellC/examples/objects.tell").read_text()), []) for fixture, expected in [("private_field", "field 'secret' is private"), ("argument_type", "expected 'number', got 'text'"), ("return_type", "expected 'number', got 'text'"), ("missing_return", "function must return a value on every path")]: with self.subTest(fixture=fixture): source = (ROOT / "TellC/tests/fixtures" / (fixture + ".tell")).read_text() - self.assertIn(expected, diagnostics(source, "file:///test.tell")[0]["message"]) + self.assertIn(expected, self.diagnostics(source)[0]["message"]) def test_stdio_lifecycle_and_live_diagnostics(self): - uri = "file:///test.tell" - requests = [ - {"id": 1, "method": "initialize", "params": {}}, - {"method": "initialized", "params": {}}, - {"method": "textDocument/didOpen", "params": {"textDocument": {"uri": uri, "text": 'let number age be "old"', "version": 1}}}, - {"method": "textDocument/didChange", "params": {"textDocument": {"uri": uri, "version": 2}, "contentChanges": [{"text": "let number age be 3"}]}}, - {"id": 2, "method": "textDocument/completion", "params": {}}, - {"method": "textDocument/didClose", "params": {"textDocument": {"uri": uri}}}, - {"id": 3, "method": "shutdown"}, {"method": "exit"}, - ] - process = subprocess.run([sys.executable, "-m", "tell_lsp"], input=b"".join(frame({"jsonrpc": "2.0", **r}) for r in requests), capture_output=True, check=True, cwd=ROOT / "lsp") - responses = messages(process.stdout) + responses = self.exchange([ + {"method": "textDocument/didOpen", "params": {"textDocument": {"uri": URI, "text": "let number age be", "version": 1}}}, + {"id": "completion", "method": "textDocument/completion", "params": {}}, + {"method": "textDocument/didChange", "params": {"textDocument": {"uri": URI, "version": 2}, "contentChanges": [{"text": "let number age be 3"}]}}, + {"method": "textDocument/didClose", "params": {"textDocument": {"uri": URI}}}, + {"id": 2, "method": "textDocument/hover", "params": {}}, + ]) self.assertEqual(responses[0]["result"]["capabilities"]["textDocumentSync"]["change"], 1) - self.assertIn("expected 'number'", responses[1]["params"]["diagnostics"][0]["message"]) - self.assertEqual(responses[2]["params"]["diagnostics"], []) - self.assertEqual(responses[2]["params"]["version"], 2) - self.assertIn("object", [item["label"] for item in responses[3]["result"]]) + self.assertIn("expected expression", responses[1]["params"]["diagnostics"][0]["message"]) + self.assertEqual(responses[1]["params"]["version"], 1) + self.assertEqual(responses[2]["id"], "completion") + self.assertIn("object", [item["label"] for item in responses[2]["result"]]) + self.assertEqual(responses[3]["params"]["diagnostics"], []) + self.assertEqual(responses[3]["params"]["version"], 2) self.assertEqual(responses[4]["params"]["diagnostics"], []) - self.assertIsNone(responses[5]["result"]) - self.assertEqual(process.stderr, b"") + self.assertEqual(responses[5]["error"]["code"], -32601) + self.assertIsNone(responses[6]["result"]) def test_utf8_compiler_column_converts_to_utf16(self): - result = diagnostics('say "😀" +\n', "file:///test.tell") - self.assertEqual(result[0]["range"]["start"], {"line": 0, "character": 10}) - - def test_missing_compiler_is_visible(self): - original = os.environ["TELL_LSP_COMPILER"] - try: - os.environ["TELL_LSP_COMPILER"] = "/nonexistent/tellc" - self.assertIn("Could not run TellC checker", diagnostics("", "file:///x.tell")[0]["message"]) - finally: - os.environ["TELL_LSP_COMPILER"] = original + result = self.diagnostics('say "ok"\r\nsay "😀" +\r\n') + self.assertEqual(result[0]["range"], { + "start": {"line": 1, "character": 10}, + "end": {"line": 1, "character": 10}, + }) if __name__ == "__main__": diff --git a/scripts/build-tellc-macos.sh b/scripts/build-tellc-macos.sh index 719ef16..a90efe4 100755 --- a/scripts/build-tellc-macos.sh +++ b/scripts/build-tellc-macos.sh @@ -92,3 +92,4 @@ cmake --build "$BUILD_ROOT/tellc" --parallel "$JOBS" ctest --test-dir "$BUILD_ROOT/tellc" --output-on-failure printf 'Standalone compiler: %s\n' "$BUILD_ROOT/tellc/tellc" +printf 'Language server: %s\n' "$BUILD_ROOT/tellc/tell-lsp" From 049ef83a26d621b5396af833e54e243cf0f0523e Mon Sep 17 00:00:00 2001 From: Neonwave175 Date: Wed, 18 Nov 2026 22:21:06 +0530 Subject: [PATCH 3/3] Document TellC collection and operator support --- README.md | 2 +- TellC/README.md | 80 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 56 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 35b6284..62cd6a0 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ say result Declarations specify a type, function and method parameters use `type: name`, and every function declares a result with `out type` (or `out void`). The compiler checks initializers, arguments, and returns, and requires a value-returning function to return on every path. Built-in types include `number` (signed 64-bit integer), `decimal` (64-bit floating point), `text`, and `bool`. Newlines end statements; semicolons can separate statements on one line. Keywords and built-in type names are case insensitive; object type names are case sensitive. -The executable backend supports typed functions, integer arithmetic, equality, `if` blocks, output and interpolation, and objects with fields and typed methods. Object members can be private (`own`) or public (`others`); see [the object example](TellC/examples/objects.tell). The parser accepts some constructs that the backend cannot yet compile, including assignment, loops, and lists. Decimal arithmetic, imports, and executable foreign calls are also unsupported. TellC reports unsupported syntax or compilation features instead of silently running them; see the [TellC guide](TellC/README.md) for the complete current scope. +TellC compiles typed functions, objects, loops, resizable typed lists, and basic console and file I/O. Object members can be private (`own`) or public (`others`), as shown in the [object example](TellC/examples/objects.tell). See the [TellC guide](TellC/README.md) for supported operations and library linking. On Apple silicon, build the standalone compiler from the repository root with CMake, Ninja, Xcode command-line tools, and a matching LLVM 24 source tree: diff --git a/TellC/README.md b/TellC/README.md index 3da0f42..ae91301 100644 --- a/TellC/README.md +++ b/TellC/README.md @@ -5,15 +5,15 @@ A C++17 compiler for an explicitly typed variant of Tell. It parses source into Platform setup, build requirements, and test coverage are documented in [the documentation index](docs/README.md), [the build guide](docs/BUILDING.md), [the testing guide](docs/TESTING.md), and [the architecture guide](docs/ARCHITECTURE.md). ```tell -let text name be "Steve Jobs"; let number age be 56 -let decimal height be 1.88 -let bool alive be false -say name +let text service_name be "Customer Portal"; let number active_users be 250 +let decimal availability_target be 99.95 +let bool release_approved be false +say service_name ``` Newlines terminate statements; semicolons separate statements on the same line. Keywords and built-in type names are case insensitive. Object type names are case sensitive. -The parser recognizes typed declarations, assignment statements, output, returns, functions, if/otherwise, while, repeat, for each, literals, lists, calls, member and index access, and common symbolic and word operators. The executable backend intentionally implements a smaller subset. Syntax outside the parser subset is reported as a parse error; syntax represented by the parser but not yet implemented by the backend is rejected during compilation with a source-located diagnostic. Natural-language comparison phrases, classes, imports, and named call arguments are not currently supported. Strings support basic escapes; source columns count bytes. +The parser recognizes typed declarations, assignment statements, output, returns, functions, if/otherwise, while, repeat, for each, literals, lists, calls, member and index access, and common symbolic and word operators. The executable backend intentionally implements a smaller subset. Syntax outside the parser subset is reported as a parse error; syntax represented by the parser but not yet implemented by the backend is rejected during compilation with a source-located diagnostic. The parser also accepts the equality phrases listed below. Classes, imports, and named call arguments are not currently supported. Strings support basic escapes; source columns count bytes. **Compiled functions and methods are strongly typed.** Parameters must use `type: name` and every function must declare its result with `out type` (use `out void` for no result). For example: @@ -32,7 +32,7 @@ On Apple silicon, build the standalone compiler from the repository root: ```sh LLVM_SOURCE=/path/to/llvm-project ./scripts/build-tellc-macos.sh -build/tellc-macos-arm64/tellc/tellc TellC/examples/objects.tell -o macbook +build/tellc-macos-arm64/tellc/tellc TellC/examples/objects.tell -o release-policy ``` The script needs CMake, Ninja, Xcode command-line tools and the matching LLVM @@ -115,31 +115,37 @@ actual library's ABI exactly. ## Executable backend -The backend currently supports typed declarations, output with `{name}` interpolation, typed functions and methods, calls, returns, object construction, field access, equality, signed integer arithmetic, `if` blocks, variable reassignment, typed lists and vectors, indexed reads and writes, and `while`, `repeat`, and `for each` loops. `number` is a signed 64-bit integer and `decimal` is a 64-bit floating-point value. Text equality compares contents; object equality compares identity. Decimal arithmetic and unary operators still require code generation support. Dictionaries, tuples, and sets are not yet implemented. +The backend supports typed declarations, functions and methods, objects, +output and interpolation, integer arithmetic, comparisons, Boolean and +bitwise operators, control flow, resizable typed lists, and basic console +and file I/O. `number` is a signed 64-bit integer and `decimal` is a 64-bit +floating-point value. Text equality compares contents and object equality +compares identity. Unary `+` and `-` work on both numeric types. Binary +decimal arithmetic, dictionaries, tuples, and sets are unsupported. ## Objects and typed function signatures Declare fields with `let TYPE self NAME be VALUE`. Members in `own` are private to methods of that object type. Members in `others` are public. ```tell -object MacBook: +object ReleasePolicy: own: - let text self password be "12345678" + let text self approved_environment be "production" others: - to check password(text: Password) out bool: - if Password = password: + to approve_environment(text: candidate_environment) out bool: + if candidate_environment = self.approved_environment: give back true end if give back false end function end object -let MacBook laptop be MacBook() -say laptop.check password("12345678") -say laptop.check password("wrong") +let ReleasePolicy policy be ReleasePolicy() +say policy.approve_environment("production") +say policy.approve_environment("staging") ``` -This program prints `true` and then `false`. `MacBook()` constructs an instance using the field initializers. Initializers run in declaration order without a `self` receiver. Instances live until the process exits. Inside methods, use `self.password` for explicit access or `password` when a local name does not shadow it. +This program prints `true` and then `false`. `ReleasePolicy()` constructs an instance using the field initializers. Initializers run in declaration order without a `self` receiver. Instances live until the process exits. Methods can read private fields with `self.field_name` or with the field name when a local name does not shadow it. Parameters use `type: name` and are separated by commas. `out` declares the return type, including `out void` for functions without a result. Arguments, initializers, and return values must match the declared types. A function with a result must return a value on every path. The parser still accepts the legacy `to name with parameter:` form for parser compatibility, but the executable backend intentionally rejects untyped function signatures. @@ -163,7 +169,7 @@ switch water_amount: ## Lists, vectors, and for each -Lists and vectors are names for the same fixed-length, homogeneous collection type. +Lists and vectors are names for the same resizable, homogeneous collection type. Use `list of TYPE` or `vector of TYPE` anywhere a type is accepted, including function signatures and object fields. They may be nested, and element types may be primitive types or objects. Literals evaluate their elements from left to right. @@ -178,14 +184,26 @@ for each score in scores: say score end for let list of text names be [] +add "Ada" to names +set names[0] to "Grace" +say len(names) # 1 +remove "Grace" from names ``` -An out-of-bounds index raises an error. `for each` evaluates its -collection expression once, visits elements in order, and runs zero times for an -empty collection. Its iteration variable is local to the loop; `break` exits the -innermost loop and `continue` advances it. An iteration variable cannot reuse an -existing local name. Collections cannot currently be resized, and -collection values cannot be printed directly. +`add VALUE to COLLECTION` appends a value. `remove VALUE from COLLECTION` +removes its first equal element, or raises a catchable error when it is absent. +`len(COLLECTION)` returns the current length. Values must match the element type; +an empty nested list passed to `add` must first be assigned to a typed variable. +Aliases and function parameters refer to the same collection, so mutations are +visible through each reference. A constant binding cannot be reassigned, but its +collection can be mutated. Collection values cannot be printed directly. + +Indexing outside the collection raises a catchable error. `for each` evaluates its +collection expression once, then checks its current length and reads the current +element at each index. Appending during iteration can extend the loop; removing +an element shifts later elements left, so a shifted element may be skipped. Its +iteration variable is local to the loop; `break` exits the innermost loop and +`continue` advances it. An iteration variable cannot reuse an existing local name. ## Reassignment and loops @@ -225,6 +243,18 @@ text, not arbitrary binary data. File errors can be caught using `try`. optional `if something goes wrong` handler receives it as `error_message`. Errors propagate through compiled functions and methods to the nearest handler; without one, the program writes an error to stderr and exits with status 1. -Conversions, input, file operations, and list index bounds produce catchable -errors. Memory allocation failures and errors from foreign libraries are not -caught. Typed catches, `finally`, and exception objects are not implemented. +Conversions, input, file operations, list index bounds, missing list items, +and list growth failures produce catchable errors. Other allocation failures +and errors from foreign libraries are not caught. Typed catches, `finally`, and +exception objects are unsupported. + +## Operators shared with the Go runtime + +TellC and the Go runtime accept `=`, `==`, `equal to`, and `is equal to` +for equality. Boolean `and` / `&&` and `or` / `||` evaluate the right +operand only when needed. Use `not` or `!` for Boolean negation. + +The integer bitwise operators are `~`, `&`, `|`, `^`, `<<`, and `>>`. +Operands and results must stay between −9,007,199,254,740,991 and +9,007,199,254,740,991, the shared exact integer range. Shift counts +must be integers from 0 to 63.