diff --git a/TellC/tests/fixtures/operators.tell b/TellC/tests/fixtures/operators.tell new file mode 100644 index 0000000..7c2771e --- /dev/null +++ b/TellC/tests/fixtures/operators.tell @@ -0,0 +1,23 @@ +say 6 & 3 +say 6 | 3 +say 6 ^ 3 +say ~5 +say 1 << 3 + 1 +say -8 >> 2 +say 1 + 2 << 2 & 15 | 1 +say 2 equal to 2 +say 2 is equal to 3 +say true && !false +say false || true +say false and (1 << 63 = 0) +say true or (1 << 63 = 0) +try: + say 1 << -1 +if something goes wrong: + say "invalid shift" +end try +try: + say 1 << 53 +if something goes wrong: + say "shift out of range" +end try diff --git a/native/internal/tell/bytecode.go b/native/internal/tell/bytecode.go index a98cf64..568487f 100644 --- a/native/internal/tell/bytecode.go +++ b/native/internal/tell/bytecode.go @@ -851,6 +851,8 @@ func foldBytecodeExpression(expr expression) expression { switch value.operator { case "not": result = !truthy(literal) + case "~": + result, err = bitwiseNot(literal) case "+", "-": var number float64 number, err = numberValue(literal) @@ -1505,6 +1507,10 @@ func executeBytecodeCode(r *runtime, instructions []BytecodeInstruction, functio case "UNARY": if instruction.Name == "not" { stack = append(stack, !truthy(value)) + } else if instruction.Name == "~" { + var result any + result, err = bitwiseNot(value) + if err == nil { stack = append(stack, result) } } else { var number float64 number, err = numberValue(value) diff --git a/native/internal/tell/expression_ast.go b/native/internal/tell/expression_ast.go index 17b7912..d382cd0 100644 --- a/native/internal/tell/expression_ast.go +++ b/native/internal/tell/expression_ast.go @@ -134,6 +134,7 @@ func (p *expressionParser) parsePrecedence(min int) (expression, error) { break } operatorToken := p.take() + if op == "=" { op = "==" } next := prec + 1 if op == "**" { next = prec @@ -169,8 +170,8 @@ func (p *expressionParser) parsePrefix() (expression, error) { switch t.kind { case "number", "string", "true", "false", "nothing": value = literalExpression{token: t} - case "-", "+", "not": - operand, err := p.parsePrecedence(7) + case "-", "+", "not", "~": + operand, err := p.parsePrecedence(10) if err != nil { return nil, err } @@ -453,6 +454,11 @@ func (r *runtime) evalExpression(expr expression, e *env) (any, error) { if x.operator == "not" { return !truthy(value), nil } + if x.operator == "~" { + result, err := bitwiseNot(value) + if err != nil { return nil, withSourceLocation(err, r.filename, x.line, x.column) } + return result, nil + } n, err := numberValue(value) if err != nil { return nil, withSourceLocation(err, r.filename, x.line, x.column) @@ -718,7 +724,38 @@ func indexValue(value, index any) (any, error) { } } +func bitwiseInteger(value any) (int64, error) { + n, err := numberValue(value) + if err != nil || math.IsNaN(n) || math.IsInf(n, 0) || math.Trunc(n) != n || n < -9007199254740991 || n > 9007199254740991 { + return 0, fail("bitwise operators require integers within the exact numeric range") + } + return int64(n), nil +} + +func bitwiseNot(value any) (any, error) { + n, err := bitwiseInteger(value) + if err != nil { return nil, err } + result := ^n + if result < -9007199254740991 || result > 9007199254740991 { return nil, fail("bitwise result exceeds exact numeric range") } + return float64(result), nil +} + func applyBinary(op string, a, b any) (any, error) { + if op == "&" || op == "|" || op == "^" || op == "<<" || op == ">>" { + left, err := bitwiseInteger(a); if err != nil { return nil, err } + right, err := bitwiseInteger(b); if err != nil { return nil, err } + var result int64 + switch op { + case "&": result = left & right + case "|": result = left | right + case "^": result = left ^ right + case "<<", ">>": + if right < 0 || right >= 64 { return nil, fail("shift count must be between 0 and 63") } + if op == "<<" { result = left << uint(right); if (result >> uint(right)) != left { return nil, fail("bitwise shift overflow") } } else { result = left >> uint(right) } + } + if result < -9007199254740991 || result > 9007199254740991 { return nil, fail("bitwise result exceeds exact numeric range") } + return float64(result), nil + } if op == "and" || op == "or" { return truthy(b), nil } diff --git a/native/internal/tell/lexer.go b/native/internal/tell/lexer.go index e92d123..a24ca4c 100644 --- a/native/internal/tell/lexer.go +++ b/native/internal/tell/lexer.go @@ -56,6 +56,7 @@ var languagePhrases = []languagePhrase{ {"is at most", "<=", "<="}, {"is not in", "notin", "notin"}, {"is equal to", "==", "=="}, + {"equal to", "==", "=="}, {"isn't", "!=", "!="}, {"is not", "!=", "!="}, {"is below", "<", "<"}, @@ -206,13 +207,16 @@ func lex(s string) ([]token, error) { } if i+1 < len(s) { two := s[i : i+2] - if two == ">=" || two == "<=" || two == "==" || two == "!=" || two == "**" || two == "//" { - appendToken(two, two, start, i+2) + if two == "&&" || two == "||" || two == "<<" || two == ">>" || two == ">=" || two == "<=" || two == "==" || two == "!=" || two == "**" || two == "//" { + kind := two + if two == "&&" { kind = "and" }; if two == "||" { kind = "or" } + appendToken(kind, two, start, i+2) i += 2 continue } } - if strings.ContainsRune("+-*/%<>()[]{}.,:", c) { + if c == '!' { appendToken("not", "!", start, start+size); i += size; continue } + if strings.ContainsRune("+-*/%<>()[]{}.,:~&|^", c) { appendToken(string(c), string(c), start, start+size) i += size continue @@ -235,14 +239,22 @@ func precedence(op string) int { return 1 case "and": return 2 - case "==", "!=", "<", ">", "<=", ">=", "in", "notin": + case "=", "==", "!=", "<", ">", "<=", ">=", "in", "notin": return 3 - case "+", "-": + case "|": return 4 - case "*", "/", "//", "%": + case "^": return 5 - case "**": + case "&": return 6 + case "<<", ">>": + return 7 + case "+", "-": + return 8 + case "*", "/", "//", "%": + return 9 + case "**": + return 11 } return 0 } diff --git a/native/internal/tell/operators_test.go b/native/internal/tell/operators_test.go new file mode 100644 index 0000000..b2c4abb --- /dev/null +++ b/native/internal/tell/operators_test.go @@ -0,0 +1,43 @@ +package tell + +import ( + "bytes" + "os" + "strings" + "testing" +) + +func TestSharedOperators(t *testing.T) { + source, err := os.ReadFile("../../../TellC/tests/fixtures/operators.tell") + if err != nil { + t.Fatal(err) + } + const want = "2\n7\n5\n-6\n16\n-2\n13\ntrue\nfalse\ntrue\ntrue\nfalse\ntrue\ninvalid shift\nshift out of range\n" + var out bytes.Buffer + if err := RunWithIO(string(source), "operators.tell", strings.NewReader(""), &out); err != nil { + t.Fatal(err) + } + if out.String() != want { + t.Fatalf("interpreter = %q, want %q", out.String(), want) + } + if got := runCompiled(t, string(source)); got != want { + t.Fatalf("bytecode = %q, want %q", got, want) + } +} + +func TestInvalidBitwiseOperands(t *testing.T) { + for _, source := range []string{"say 1 << -1", "say 1 << 64", "say 1.5 & 1", "say 1 << 53", "say ~9007199254740991"} { + t.Run(source, func(t *testing.T) { + if err := RunWithIO(source, "invalid.tell", strings.NewReader(""), &bytes.Buffer{}); err == nil { + t.Fatal("expected error") + } + artifact, err := Compile(source, "invalid.tell") + if err != nil { + return + } + if err = RunBytecode(artifact, "invalid.tbc", strings.NewReader(""), &bytes.Buffer{}); err == nil { + t.Fatal("expected bytecode error") + } + }) + } +} diff --git a/native/internal/tell/parser.go b/native/internal/tell/parser.go index 924233f..06674f7 100644 --- a/native/internal/tell/parser.go +++ b/native/internal/tell/parser.go @@ -9,7 +9,6 @@ import ( "unicode/utf8" ) - const maxLoopIterations = 1_000_000 const maxFunctionCallDepth = 64 @@ -24,11 +23,11 @@ func DefaultExecutionLimits() ExecutionLimits { } type Error struct { - Kind string - Message string + Kind string + Message string Filename string - Line int - Column int + Line int + Column int } // tupleValue is an immutable, ordered Tell sequence distinct from mutable lists. @@ -50,12 +49,16 @@ func fail(format string, args ...any) error { // Nested expression errors keep their more precise location. func ioFailure(err error) error { - if err == nil { return nil } - return &Error{Kind:"IOError", Message:err.Error()} + if err == nil { + return nil + } + return &Error{Kind: "IOError", Message: err.Error()} } func ioFailureAt(err error, filename string, expression expression) error { - if err == nil { return nil } + if err == nil { + return nil + } line, column := expressionPosition(expression) if line > 0 && column > 0 { return withSourceLocation(ioFailure(err), filename, line, column) @@ -77,13 +80,13 @@ func withSourceLocation(err error, filename string, line, column int) error { return err } } - return &Error{Kind:kind, Message:message, Filename:filename, Line:line, Column:column} + return &Error{Kind: kind, Message: message, Filename: filename, Line: line, Column: column} } type node struct { kind, text string line, column int - conditionColumnOffset int + conditionColumnOffset int condition, name, base string conditionExpr expression params []string @@ -340,18 +343,26 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { {"key ", " in "}, } { argument, isPhrase := prefixFold(name, phrase.prefix) - if !isPhrase { continue } + if !isPhrase { + continue + } indexText, collectionName, hasIn := splitFoldOnce(argument, phrase.delimiter) - if !hasIn || !identifier(collectionName) { continue } + if !hasIn || !identifier(collectionName) { + continue + } parsedIndex, validIndex := parseExpression(indexText) parsedValue, validValue := parseExpression(expressionText) - if !validIndex || !validValue { continue } + if !validIndex || !validValue { + continue + } containerOffset := strings.LastIndex(source, collectionName) - if containerOffset < 0 { containerOffset = 0 } + if containerOffset < 0 { + containerOffset = 0 + } targetColumn := column + utf8.RuneCountInString(source[:containerOffset]) - containerExpr := nameExpression{name:collectionName, line:line, column:targetColumn} - target := indexExpression{value:containerExpr, index:parsedIndex, line:line, column:targetColumn} - return &simpleStatement{kind:"set_index", source:name, target:target, expression:parsedValue, hasExpression:true} + containerExpr := nameExpression{name: collectionName, line: line, column: targetColumn} + target := indexExpression{value: containerExpr, index: parsedIndex, line: line, column: targetColumn} + return &simpleStatement{kind: "set_index", source: name, target: target, expression: parsedValue, hasExpression: true} } if strings.Contains(name, ",") { if names, valid := parseLoopNames(name); valid && len(names) > 1 { @@ -379,9 +390,13 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { } for _, prefix := range []string{"call ", "run "} { rest, ok := prefixFold(source, prefix) - if !ok { continue } + if !ok { + continue + } name, tail := firstWord(rest) - if !identifier(name) { continue } + if !identifier(name) { + continue + } arguments := []callArgument{} valid := true if strings.TrimSpace(tail) != "" { @@ -391,26 +406,33 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { } else { for _, argument := range splitAnd(argumentText) { parsed, parsedOK := parseExpression(argument) - if !parsedOK { valid = false; break } - arguments = append(arguments, callArgument{value:parsed}) + if !parsedOK { + valid = false + break + } + arguments = append(arguments, callArgument{value: parsed}) } } } if valid { nameOffset := strings.Index(source, name) - if nameOffset < 0 { nameOffset = len(prefix) } - callee := nameExpression{name:name, line:line, column:column+utf8.RuneCountInString(source[:nameOffset])} + if nameOffset < 0 { + nameOffset = len(prefix) + } + callee := nameExpression{name: name, line: line, column: column + utf8.RuneCountInString(source[:nameOffset])} callOffset := strings.Index(strings.ToLower(source), strings.TrimSpace(prefix)) - if callOffset < 0 { callOffset = 0 } - call := callExpression{callee:callee, args:arguments, line:line, column:column+utf8.RuneCountInString(source[:callOffset])} - return &simpleStatement{kind:"call", source:source, expression:call, hasExpression:true} + if callOffset < 0 { + callOffset = 0 + } + call := callExpression{callee: callee, args: arguments, line: line, column: column + utf8.RuneCountInString(source[:callOffset])} + return &simpleStatement{kind: "call", source: source, expression: call, hasExpression: true} } } if rest, ok := prefixFold(source, "remember "); ok { expressionText, field, separated := splitFoldOnce(rest, " as this.") if separated && identifier(field) { if parsed, valid := parseExpression(expressionText); valid { - return &simpleStatement{kind:"remember_field", source:expressionText, names:[]string{field}, expression:parsed, hasExpression:true} + return &simpleStatement{kind: "remember_field", source: expressionText, names: []string{field}, expression: parsed, hasExpression: true} } } } @@ -450,7 +472,6 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { return statement } - for _, operation := range []struct{ prefix, delimiter, kind string }{ {"add ", " to ", "add"}, {"remove ", " from ", "remove"}, @@ -459,7 +480,7 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { valueText, targetName, separated := splitFoldOnce(rest, operation.delimiter) if separated && identifier(targetName) { if parsed, valid := parseExpression(valueText); valid { - return &simpleStatement{kind:operation.kind, names:[]string{targetName}, expression:parsed, hasExpression:true} + return &simpleStatement{kind: operation.kind, names: []string{targetName}, expression: parsed, hasExpression: true} } } } @@ -468,7 +489,7 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { pathText, alias, separated := splitFoldOnce(rest, " as ") if separated && identifier(alias) { if parsed, valid := parseExpression(pathText); valid { - return &simpleStatement{kind:"import_module", names:[]string{alias}, expression:parsed, hasExpression:true} + return &simpleStatement{kind: "import_module", names: []string{alias}, expression: parsed, hasExpression: true} } } } @@ -477,9 +498,11 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { moduleName, alias, hasAlias := splitFoldOnce(rest, " as ") moduleName = strings.TrimSpace(moduleName) if _, err := newPythonModule(moduleName); err == nil { - if !hasAlias { alias = moduleName[strings.LastIndex(moduleName, ".")+1:] } + if !hasAlias { + alias = moduleName[strings.LastIndex(moduleName, ".")+1:] + } if identifier(alias) { - return &simpleStatement{kind:"import_python", source:moduleName, names:[]string{alias}} + return &simpleStatement{kind: "import_python", source: moduleName, names: []string{alias}} } } } @@ -487,9 +510,11 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { moduleName, alias, hasAlias := splitFoldOnce(rest, " as ") moduleName = strings.TrimSpace(moduleName) if module, err := newJavaScriptModule(moduleName); err == nil { - if !hasAlias { alias = defaultJavaScriptAlias(moduleName) } + if !hasAlias { + alias = defaultJavaScriptAlias(moduleName) + } if identifier(alias) { - return &simpleStatement{kind:"import_javascript", source:module.name, names:[]string{alias}} + return &simpleStatement{kind: "import_javascript", source: module.name, names: []string{alias}} } } } @@ -505,12 +530,14 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { } library := strings.ToLower(strings.ReplaceAll(requested, " ", "")) switch library { - case "mathematics": library = "math" - case "dateandtime", "dates": library = "datetime" + case "mathematics": + library = "math" + case "dateandtime", "dates": + library = "datetime" } if requested != "" && aliasOK && identifier(strings.ReplaceAll(requested, " ", "")) { if _, err := moduleValue(library); err == nil { - return &simpleStatement{kind:"import_library", source:library, names:[]string{alias}} + return &simpleStatement{kind: "import_library", source: library, names: []string{alias}} } } } @@ -520,7 +547,7 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { prompt, name, separated := splitFoldOnce(rest, delimiter) if separated && identifier(name) { if parsed, valid := parseExpression(prompt); valid { - return &simpleStatement{kind:"ask", names:[]string{name}, expression:parsed, hasExpression:true} + return &simpleStatement{kind: "ask", names: []string{name}, expression: parsed, hasExpression: true} } } } @@ -529,7 +556,7 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { pathText, name, separated := splitFoldOnce(rest, " into ") if separated && identifier(name) { if parsed, valid := parseExpression(pathText); valid { - return &simpleStatement{kind:"read_file", names:[]string{name}, expression:parsed, hasExpression:true} + return &simpleStatement{kind: "read_file", names: []string{name}, expression: parsed, hasExpression: true} } } } @@ -543,7 +570,7 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { valueExpr, validValue := parseExpression(valueText) pathExpr, validPath := parseExpression(pathText) if validValue && validPath { - return &simpleStatement{kind:operation.kind, expression:valueExpr, target:pathExpr, hasExpression:true, hasTarget:true} + return &simpleStatement{kind: operation.kind, expression: valueExpr, target: pathExpr, hasExpression: true, hasTarget: true} } } } @@ -577,10 +604,10 @@ func parseSimpleStatement(source string, line, column int) *simpleStatement { } } } - if strings.EqualFold(source, "stop this loop") || strings.EqualFold(source, "break out of this loop") { + if strings.EqualFold(source, "break") || strings.EqualFold(source, "stop this loop") || strings.EqualFold(source, "break out of this loop") { return &simpleStatement{kind: "break"} } - if strings.EqualFold(source, "skip to the next repetition") || strings.EqualFold(source, "continue to the next repetition") { + if strings.EqualFold(source, "continue") || strings.EqualFold(source, "skip to the next repetition") || strings.EqualFold(source, "continue to the next repetition") { return &simpleStatement{kind: "continue"} } if source != "" { @@ -802,6 +829,9 @@ func stripOptionalThen(condition string) string { func normalizeBlockLine(line string) string { normalized := strings.ToLower(strings.TrimSpace(line)) + if normalized == "end for" || normalized == "end for." { + return "end loop." + } if normalized == "else:" { return "otherwise:" } @@ -1170,4 +1200,3 @@ func splitTop(s, sep string) []string { } return append(out, strings.TrimSpace(s[start:])) } - diff --git a/native/internal/tell/tellc_syntax_test.go b/native/internal/tell/tellc_syntax_test.go new file mode 100644 index 0000000..f3150d7 --- /dev/null +++ b/native/internal/tell/tellc_syntax_test.go @@ -0,0 +1,32 @@ +package tell + +import ( + "bytes" + "strings" + "testing" +) + +func TestTellCSyntax(t *testing.T) { + source := `to factorial(number: n) out number: + if n = 0: + return 1 + end if + return n * factorial(n - 1) +end function +let number result be factorial(5) +let bool correct be result = 120 +say correct +for each value in [1, 2, 3]: + if value = 2: + continue + end if + say value +end for` + var out bytes.Buffer + if err := RunWithIO(source, "tellc.tell", strings.NewReader(""), &out); err != nil { + t.Fatal(err) + } + if got, want := out.String(), "true\n1\n3\n"; got != want { + t.Fatalf("got %q, want %q", got, want) + } +}