Correct begin and end (line+char) positions for each token and node

lib/rkelly/char_pos.rb

@@ -0,0 +1,35 @@
+
+module RKelly
+  # Represents a character position in source code.
+  #
+  # It's a value object - it can't be modified.
+  class CharPos
+    attr_reader :line, :char, :index
+
+    def initialize(line, char, index)
+      @line = line
+      @char = char
+      @index = index
+    end
+
+    # Creates a new character position that's a given string away from
+    # this one.
+    def next(string)
+      if string.include?("\n")
+        lines = string.split(/\n/, -1)
+        CharPos.new(@line + lines.length - 1, lines.last.length, @index + string.length)
+      else
+        CharPos.new(@line, @char + string.length, @index + string.length)
+      end
+    end
+
+    def to_s
+      "{line:#{@line} char:#{@char} (#{@index})}"
+    end
+
+    alias_method :inspect, :to_s
+
+    # A re-usable empty position
+    EMPTY = CharPos.new(1,0,-1)
+  end
+end

lib/rkelly/char_range.rb

@@ -0,0 +1,31 @@
+require 'rkelly/char_pos'
+
+module RKelly
+  # Represents a syntax element location in source code - where it
+  # begins and where it ends.
+  #
+  # It's a value object - it can't be modified.
+  class CharRange
+    attr_reader :from, :to
+
+    def initialize(from, to)
+      @from = from
+      @to = to
+    end
+
+    # Creates a new range that immediately follows this one and
+    # contains the given string.
+    def next(string)
+      CharRange.new(@to.next(string.slice(0, 1)), @to.next(string))
+    end
+
+    def to_s
+      "<#{@from}...#{@to}>"
+    end
+
+    alias_method :inspect, :to_s
+
+    # A re-usable empty range
+    EMPTY = CharRange.new(CharPos::EMPTY, CharPos::EMPTY)
+  end
+end

lib/rkelly/nodes/node.rb

@@ -5,11 +5,17 @@ class Node
       include RKelly::Visitors
       include Enumerable
 
-      attr_accessor :value, :comments, :line, :filename
+      attr_accessor :value, :comments, :range, :filename
       def initialize(value)
         @value = value
         @comments = []
-        @filename = @line = nil
+        @range = CharRange::EMPTY
+        @filename = nil
+      end
+
+      # For backwards compatibility
+      def line
+        @range.from.line
       end
 
       def ==(other)

lib/rkelly/parser.rb

@@ -13,8 +13,12 @@ def #{im}(val, _values, result)
           r = super(val.map { |v|
               v.is_a?(Token) ? v.to_racc_token[1] : v
             }, _values, result)
-          if token = val.find { |v| v.is_a?(Token) }
-            r.line = token.line if r.respond_to?(:line)
+
+          suitable_values = val.find_all {|v| v.is_a?(Node) || v.is_a?(Token) }
+          first = suitable_values.first
+          last = suitable_values.last
+          if first
+            r.range = CharRange.new(first.range.from, last.range.to) if r.respond_to?(:range)
             r.filename = @filename if r.respond_to?(:filename)
           end
           r
@@ -37,34 +41,15 @@ def parse(javascript, filename = nil)
       @position = 0
       @filename = filename
       ast = do_parse
-      apply_comments(ast)
+      ast.comments = @comments if ast
+      ast
     end
 
     def yyabort
       raise "something bad happened, please report a bug with sample JavaScript"
     end
 
     private
-    def apply_comments(ast)
-      ast_hash = Hash.new { |h,k| h[k] = [] }
-      (ast || []).each { |n|
-        next unless n.line
-        ast_hash[n.line] << n
-      }
-      max = ast_hash.keys.sort.last
-      @comments.each do |comment|
-        node = nil
-        comment.line.upto(max) do |line|
-          if ast_hash.key?(line)
-            node = ast_hash[line].first
-            break
-          end
-        end
-        node.comments << comment if node
-      end if max
-      ast
-    end
-
     def on_error(error_token_id, error_value, value_stack)
       if logger
         logger.error(token_to_str(error_token_id))

lib/rkelly/token.rb

@@ -1,12 +1,17 @@
 module RKelly
   class Token
-    attr_accessor :name, :value, :transformer, :line
+    attr_accessor :name, :value, :transformer, :range
     def initialize(name, value, &transformer)
       @name         = name
       @value        = value
       @transformer  = transformer
     end
 
+    # For backwards compatibility
+    def line
+      @range.from.line
+    end
+
     def to_racc_token
       return transformer.call(name, value) if transformer
       [name, value]

lib/rkelly/tokenizer.rb

@@ -1,12 +1,13 @@
 require 'rkelly/lexeme'
+require 'rkelly/char_range'
 require 'strscan'
 
 module RKelly
   class Tokenizer
     KEYWORDS = %w{
       break case catch continue default delete do else finally for function
-      if in instanceof new return switch this throw try typeof var void while 
-      with 
+      if in instanceof new return switch this throw try typeof var void while
+      with
 
       const true false null debugger
     }
@@ -115,7 +116,7 @@ def tokenize(string)
     def raw_tokens(string)
       scanner = StringScanner.new(string)
       tokens = []
-      line_number = 1
+      range = CharRange::EMPTY
       accepting_regexp = true
       while !scanner.eos?
         longest_token = nil
@@ -134,14 +135,14 @@ def raw_tokens(string)
           accepting_regexp = followable_by_regex(longest_token)
         end
 
-        longest_token.line = line_number
-        line_number += longest_token.value.scan(/\n/).length
+        range = range.next(longest_token.value)
         scanner.pos += longest_token.value.length
+        longest_token.range = range
         tokens << longest_token
       end
       tokens
     end
-  
+
     private
     def token(name, pattern = nil, &block)
       @lexemes << Lexeme.new(name, pattern, &block)

test/test_char_pos.rb

@@ -0,0 +1,39 @@
+require File.dirname(__FILE__) + "/helper"
+
+class CharPosTest < NodeTestCase
+  def test_advancing_empty_position
+    a = RKelly::CharPos::EMPTY
+    b = a.next("foo")
+
+    assert_equal(1, b.line)
+    assert_equal(3, b.char)
+    assert_equal(2, b.index)
+  end
+
+  def test_advancing_with_single_line_string
+    a = RKelly::CharPos.new(3,5,22)
+    b = a.next("foo bar")
+
+    assert_equal(3, b.line)
+    assert_equal(12, b.char)
+    assert_equal(29, b.index)
+  end
+
+  def test_advancing_with_multi_line_string
+    a = RKelly::CharPos.new(3,5,22)
+    b = a.next("\nfoo\nbar\nbaz")
+
+    assert_equal(6, b.line)
+    assert_equal(3, b.char)
+    assert_equal(34, b.index)
+  end
+
+  def test_advancing_with_multi_line_string_ending_with_newline
+    a = RKelly::CharPos.new(3,5,22)
+    b = a.next("\nfoo\nbar\n")
+
+    assert_equal(6, b.line)
+    assert_equal(0, b.char)
+    assert_equal(31, b.index)
+  end
+end

test/test_char_range.rb

@@ -0,0 +1,29 @@
+require File.dirname(__FILE__) + "/helper"
+
+class CharRangeTest < NodeTestCase
+  def test_advancing_empty_range
+    a = RKelly::CharRange::EMPTY
+    b = a.next("foo")
+
+    assert_equal(1, b.from.line)
+    assert_equal(1, b.from.char)
+    assert_equal(0, b.from.index)
+
+    assert_equal(1, b.to.line)
+    assert_equal(3, b.to.char)
+    assert_equal(2, b.to.index)
+  end
+
+  def test_advancing_with_multiline_string
+    a = RKelly::CharRange.new(RKelly::CharPos.new(1,1,0), RKelly::CharPos.new(1,1,0))
+    b = a.next("foo\nblah")
+
+    assert_equal(1, b.from.line)
+    assert_equal(2, b.from.char)
+    assert_equal(1, b.from.index)
+
+    assert_equal(2, b.to.line)
+    assert_equal(4, b.to.char)
+    assert_equal(8, b.to.index)
+  end
+end

test/test_comments.rb

@@ -13,17 +13,14 @@ def test_some_comments
       }
     eojs
 
-    func = ast.pointcut(FunctionDeclNode).matches.first
-    assert func
-    assert_match('awesome', func.comments[0].value)
-    assert_match('side', func.comments[1].value)
-
-    return_node = ast.pointcut(ReturnNode).matches.first
-    assert return_node
-    assert_match('America', return_node.comments[0].value)
+    assert ast
+    assert_equal(3, ast.comments.length)
+    assert_match('awesome', ast.comments[0].value)
+    assert_match('side', ast.comments[1].value)
+    assert_match('America', ast.comments[2].value)
   end
 
-  def test_even_more_comments
+  def test_only_comments
     parser = RKelly::Parser.new
     ast = parser.parse(<<-eojs)
       /**
@@ -32,13 +29,17 @@ def test_even_more_comments
       /**
        * This is an awesome test comment.
        */
-      function aaron() { // This is a side comment
-        var x = 10;
-        return 1 + 1; // America!
-      }
     eojs
-    func = ast.pointcut(FunctionDeclNode).matches.first
-    assert func
-    assert_equal(3, func.comments.length)
+
+    assert ast
+    assert_equal(2, ast.comments.length)
+  end
+
+  def test_empty_source_results_in_zero_comments
+    parser = RKelly::Parser.new
+    ast = parser.parse("")
+
+    assert ast
+    assert_equal(0, ast.comments.length)
   end
 end

test/test_line_number.rb

@@ -20,4 +20,24 @@ def test_line_numbers
     assert return_node
     assert_equal(6, return_node.line)
   end
+
+  def test_ranges
+    parser = RKelly::Parser.new
+    ast = parser.parse(<<-eojs)
+      /**
+       * This is an awesome test comment.
+       */
+      function aaron() {
+        var x = 10;
+        return 1 + 1;
+      }
+    eojs
+    func = ast.pointcut(FunctionDeclNode).matches.first
+    assert func
+    assert_equal("<{line:4 char:7 (68)}...{line:7 char:7 (135)}>", func.range.to_s)
+
+    return_node = ast.pointcut(ReturnNode).matches.first
+    assert return_node
+    assert_equal("<{line:6 char:9 (115)}...{line:6 char:21 (127)}>", return_node.range.to_s)
+  end
 end