diff --git a/lib/rkelly/char_pos.rb b/lib/rkelly/char_pos.rb new file mode 100644 index 0000000..57ae7ed --- /dev/null +++ b/lib/rkelly/char_pos.rb @@ -0,0 +1,35 @@ + +module RKelly + # Represents a character position in source code. + # + # It's a value object - it can't be modified. + class CharPos + attr_reader :line, :char, :index + + def initialize(line, char, index) + @line = line + @char = char + @index = index + end + + # Creates a new character position that's a given string away from + # this one. + def next(string) + if string.include?("\n") + lines = string.split(/\n/, -1) + CharPos.new(@line + lines.length - 1, lines.last.length, @index + string.length) + else + CharPos.new(@line, @char + string.length, @index + string.length) + end + end + + def to_s + "{line:#{@line} char:#{@char} (#{@index})}" + end + + alias_method :inspect, :to_s + + # A re-usable empty position + EMPTY = CharPos.new(1,0,-1) + end +end diff --git a/lib/rkelly/char_range.rb b/lib/rkelly/char_range.rb new file mode 100644 index 0000000..16c574d --- /dev/null +++ b/lib/rkelly/char_range.rb @@ -0,0 +1,31 @@ +require 'rkelly/char_pos' + +module RKelly + # Represents a syntax element location in source code - where it + # begins and where it ends. + # + # It's a value object - it can't be modified. + class CharRange + attr_reader :from, :to + + def initialize(from, to) + @from = from + @to = to + end + + # Creates a new range that immediately follows this one and + # contains the given string. + def next(string) + CharRange.new(@to.next(string.slice(0, 1)), @to.next(string)) + end + + def to_s + "<#{@from}...#{@to}>" + end + + alias_method :inspect, :to_s + + # A re-usable empty range + EMPTY = CharRange.new(CharPos::EMPTY, CharPos::EMPTY) + end +end diff --git a/lib/rkelly/nodes/node.rb b/lib/rkelly/nodes/node.rb index dd5dcde..88cbb7f 100644 --- a/lib/rkelly/nodes/node.rb +++ b/lib/rkelly/nodes/node.rb @@ -5,11 +5,17 @@ class Node include RKelly::Visitors include Enumerable - attr_accessor :value, :comments, :line, :filename + attr_accessor :value, :comments, :range, :filename def initialize(value) @value = value @comments = [] - @filename = @line = nil + @range = CharRange::EMPTY + @filename = nil + end + + # For backwards compatibility + def line + @range.from.line end def ==(other) diff --git a/lib/rkelly/parser.rb b/lib/rkelly/parser.rb index 920de83..2247853 100644 --- a/lib/rkelly/parser.rb +++ b/lib/rkelly/parser.rb @@ -13,8 +13,12 @@ def #{im}(val, _values, result) r = super(val.map { |v| v.is_a?(Token) ? v.to_racc_token[1] : v }, _values, result) - if token = val.find { |v| v.is_a?(Token) } - r.line = token.line if r.respond_to?(:line) + + suitable_values = val.find_all {|v| v.is_a?(Node) || v.is_a?(Token) } + first = suitable_values.first + last = suitable_values.last + if first + r.range = CharRange.new(first.range.from, last.range.to) if r.respond_to?(:range) r.filename = @filename if r.respond_to?(:filename) end r @@ -37,7 +41,8 @@ def parse(javascript, filename = nil) @position = 0 @filename = filename ast = do_parse - apply_comments(ast) + ast.comments = @comments if ast + ast end def yyabort @@ -45,26 +50,6 @@ def yyabort end private - def apply_comments(ast) - ast_hash = Hash.new { |h,k| h[k] = [] } - (ast || []).each { |n| - next unless n.line - ast_hash[n.line] << n - } - max = ast_hash.keys.sort.last - @comments.each do |comment| - node = nil - comment.line.upto(max) do |line| - if ast_hash.key?(line) - node = ast_hash[line].first - break - end - end - node.comments << comment if node - end if max - ast - end - def on_error(error_token_id, error_value, value_stack) if logger logger.error(token_to_str(error_token_id)) diff --git a/lib/rkelly/token.rb b/lib/rkelly/token.rb index ce7bf34..2ea57cf 100644 --- a/lib/rkelly/token.rb +++ b/lib/rkelly/token.rb @@ -1,12 +1,17 @@ module RKelly class Token - attr_accessor :name, :value, :transformer, :line + attr_accessor :name, :value, :transformer, :range def initialize(name, value, &transformer) @name = name @value = value @transformer = transformer end + # For backwards compatibility + def line + @range.from.line + end + def to_racc_token return transformer.call(name, value) if transformer [name, value] diff --git a/lib/rkelly/tokenizer.rb b/lib/rkelly/tokenizer.rb index 296aa81..97e0e33 100644 --- a/lib/rkelly/tokenizer.rb +++ b/lib/rkelly/tokenizer.rb @@ -1,12 +1,13 @@ require 'rkelly/lexeme' +require 'rkelly/char_range' require 'strscan' module RKelly class Tokenizer KEYWORDS = %w{ break case catch continue default delete do else finally for function - if in instanceof new return switch this throw try typeof var void while - with + if in instanceof new return switch this throw try typeof var void while + with const true false null debugger } @@ -115,7 +116,7 @@ def tokenize(string) def raw_tokens(string) scanner = StringScanner.new(string) tokens = [] - line_number = 1 + range = CharRange::EMPTY accepting_regexp = true while !scanner.eos? longest_token = nil @@ -134,14 +135,14 @@ def raw_tokens(string) accepting_regexp = followable_by_regex(longest_token) end - longest_token.line = line_number - line_number += longest_token.value.scan(/\n/).length + range = range.next(longest_token.value) scanner.pos += longest_token.value.length + longest_token.range = range tokens << longest_token end tokens end - + private def token(name, pattern = nil, &block) @lexemes << Lexeme.new(name, pattern, &block) diff --git a/test/test_char_pos.rb b/test/test_char_pos.rb new file mode 100644 index 0000000..d053d74 --- /dev/null +++ b/test/test_char_pos.rb @@ -0,0 +1,39 @@ +require File.dirname(__FILE__) + "/helper" + +class CharPosTest < NodeTestCase + def test_advancing_empty_position + a = RKelly::CharPos::EMPTY + b = a.next("foo") + + assert_equal(1, b.line) + assert_equal(3, b.char) + assert_equal(2, b.index) + end + + def test_advancing_with_single_line_string + a = RKelly::CharPos.new(3,5,22) + b = a.next("foo bar") + + assert_equal(3, b.line) + assert_equal(12, b.char) + assert_equal(29, b.index) + end + + def test_advancing_with_multi_line_string + a = RKelly::CharPos.new(3,5,22) + b = a.next("\nfoo\nbar\nbaz") + + assert_equal(6, b.line) + assert_equal(3, b.char) + assert_equal(34, b.index) + end + + def test_advancing_with_multi_line_string_ending_with_newline + a = RKelly::CharPos.new(3,5,22) + b = a.next("\nfoo\nbar\n") + + assert_equal(6, b.line) + assert_equal(0, b.char) + assert_equal(31, b.index) + end +end diff --git a/test/test_char_range.rb b/test/test_char_range.rb new file mode 100644 index 0000000..508e3f3 --- /dev/null +++ b/test/test_char_range.rb @@ -0,0 +1,29 @@ +require File.dirname(__FILE__) + "/helper" + +class CharRangeTest < NodeTestCase + def test_advancing_empty_range + a = RKelly::CharRange::EMPTY + b = a.next("foo") + + assert_equal(1, b.from.line) + assert_equal(1, b.from.char) + assert_equal(0, b.from.index) + + assert_equal(1, b.to.line) + assert_equal(3, b.to.char) + assert_equal(2, b.to.index) + end + + def test_advancing_with_multiline_string + a = RKelly::CharRange.new(RKelly::CharPos.new(1,1,0), RKelly::CharPos.new(1,1,0)) + b = a.next("foo\nblah") + + assert_equal(1, b.from.line) + assert_equal(2, b.from.char) + assert_equal(1, b.from.index) + + assert_equal(2, b.to.line) + assert_equal(4, b.to.char) + assert_equal(8, b.to.index) + end +end diff --git a/test/test_comments.rb b/test/test_comments.rb index 18f106a..8ebe07a 100644 --- a/test/test_comments.rb +++ b/test/test_comments.rb @@ -13,17 +13,14 @@ def test_some_comments } eojs - func = ast.pointcut(FunctionDeclNode).matches.first - assert func - assert_match('awesome', func.comments[0].value) - assert_match('side', func.comments[1].value) - - return_node = ast.pointcut(ReturnNode).matches.first - assert return_node - assert_match('America', return_node.comments[0].value) + assert ast + assert_equal(3, ast.comments.length) + assert_match('awesome', ast.comments[0].value) + assert_match('side', ast.comments[1].value) + assert_match('America', ast.comments[2].value) end - def test_even_more_comments + def test_only_comments parser = RKelly::Parser.new ast = parser.parse(<<-eojs) /** @@ -32,13 +29,17 @@ def test_even_more_comments /** * This is an awesome test comment. */ - function aaron() { // This is a side comment - var x = 10; - return 1 + 1; // America! - } eojs - func = ast.pointcut(FunctionDeclNode).matches.first - assert func - assert_equal(3, func.comments.length) + + assert ast + assert_equal(2, ast.comments.length) + end + + def test_empty_source_results_in_zero_comments + parser = RKelly::Parser.new + ast = parser.parse("") + + assert ast + assert_equal(0, ast.comments.length) end end diff --git a/test/test_line_number.rb b/test/test_line_number.rb index cb2cb42..a378e80 100644 --- a/test/test_line_number.rb +++ b/test/test_line_number.rb @@ -20,4 +20,24 @@ def test_line_numbers assert return_node assert_equal(6, return_node.line) end + + def test_ranges + parser = RKelly::Parser.new + ast = parser.parse(<<-eojs) + /** + * This is an awesome test comment. + */ + function aaron() { + var x = 10; + return 1 + 1; + } + eojs + func = ast.pointcut(FunctionDeclNode).matches.first + assert func + assert_equal("<{line:4 char:7 (68)}...{line:7 char:7 (135)}>", func.range.to_s) + + return_node = ast.pointcut(ReturnNode).matches.first + assert return_node + assert_equal("<{line:6 char:9 (115)}...{line:6 char:21 (127)}>", return_node.range.to_s) + end end