Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions lexer/lexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,11 @@ func (l *Lexer) NextToken() Item {
if l.ch == '/' && l.peekChar() == '*' {
return l.readBlockComment()
}
// Unicode minus (U+2212) is treated as starting a line comment
// ClickHouse doesn't recognize it as an operator
if l.ch == '\u2212' {
return l.readUnicodeMinusComment()
}

switch l.ch {
case '+':
Expand Down Expand Up @@ -227,9 +232,6 @@ func (l *Lexer) NextToken() Item {
return l.readQuotedIdentifier()
case '\u201C', '\u201D': // Unicode curly double quotes " "
return l.readUnicodeQuotedIdentifier(l.ch)
case '\u2212': // Unicode minus sign −
l.readChar()
return Item{Token: token.MINUS, Value: "−", Pos: pos}
case '`':
return l.readBacktickIdentifier()
case '@':
Expand Down Expand Up @@ -297,6 +299,22 @@ func (l *Lexer) readHashComment() Item {
return Item{Token: token.COMMENT, Value: sb.String(), Pos: pos}
}

// readUnicodeMinusComment reads from a unicode minus (U+2212) to the end of line or semicolon.
// ClickHouse doesn't recognize unicode minus as an operator, so we treat it as a comment.
func (l *Lexer) readUnicodeMinusComment() Item {
pos := l.pos
var sb strings.Builder
// Skip −
sb.WriteRune(l.ch)
l.readChar()

for l.ch != '\n' && l.ch != ';' && l.ch != 0 && !l.eof {
sb.WriteRune(l.ch)
l.readChar()
}
return Item{Token: token.COMMENT, Value: sb.String(), Pos: pos}
}

func (l *Lexer) readBlockComment() Item {
pos := l.pos
var sb strings.Builder
Expand Down
2 changes: 1 addition & 1 deletion parser/testdata/02869_unicode_minus/metadata.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"todo": true}
{}