Index: src/org/mozilla/javascript/Parser.java
===================================================================
RCS file: /cvsroot/mozilla/js/rhino/src/org/mozilla/javascript/Parser.java,v
retrieving revision 1.106
diff -u -r1.106 Parser.java
--- src/org/mozilla/javascript/Parser.java	10 Nov 2006 15:27:37 -0000	1.106
+++ src/org/mozilla/javascript/Parser.java	20 Nov 2006 10:32:24 -0000
@@ -46,7 +46,10 @@
 
 import java.io.Reader;
 import java.io.IOException;
+import java.io.StringReader;
 import java.util.Hashtable;
+import java.util.List;
+import java.util.ArrayList;
 
 /**
  * This class implements the JavaScript parser.
@@ -276,6 +279,56 @@
         loopAndSwitchSet.pop();
     }
 
+    /**
+     * Parse the given sourceReader into an array of {@link Token} objects.
+     * This can be useful for inspecting a JavaScript resource for other goals
+     * than compiling and evaluating it, as the tree generated by
+     * {@link #parse(java.io.Reader, String, int)} is missing a
+     * lot of information.
+     * @param sourceString a JavaScript string
+     * @param sourceURI the source name
+     * @param lineno the line number of the first line
+     * @return an array of {@link Token} objects
+     * @throws IOException if an I/O error was encountered
+     */
+    public Token[] parseTokens(String sourceString,
+                               String sourceURI, int lineno)
+        throws IOException
+    {
+        return parseTokens(new StringReader(sourceString),
+                           sourceURI, lineno);
+    }
+
+    /**
+     * Parse the given sourceReader into an array of {@link Token} objects.
+     * This can be useful for inspecting a JavaScript resource for other goals
+     * than compiling and evaluating it, as the tree generated by
+     * {@link #parse(java.io.Reader, String, int)} is missing a
+     * lot of information.
+     * @param sourceReader the reader
+     * @param sourceURI the source name
+     * @param lineno the line number of the first line
+     * @return an array of {@link Token} objects
+     * @throws IOException if an I/O error was encountered
+     */
+    public Token[] parseTokens(Reader sourceReader,
+                               String sourceURI, int lineno)
+        throws IOException
+    {
+        this.sourceURI = sourceURI;
+        this.ts = new TokenStream(this, sourceReader, null, lineno);
+        List list = new ArrayList();
+        while (!ts.eof()) {
+            try {
+                list.add(new Token(ts.getToken(), ts.getString(),
+                                   ts.getLineno(), ts.getOffset()));
+            } catch (EvaluatorException x) {
+                // Regular expressions need special treatment by the parser - ignore.
+            }
+        }
+        return (Token[]) list.toArray(new Token[list.size()]);
+    }
+
     /*
      * Build a parse tree from the given sourceString.
      *
@@ -298,7 +351,7 @@
     }
 
     /*
-     * Build a parse tree from the given sourceString.
+     * Build a parse tree from the given sourceReader.
      *
      * @return an Object representing the parsed
      * program.  If the parse fails, null will be returned.  (The
Index: src/org/mozilla/javascript/Token.java
===================================================================
RCS file: /cvsroot/mozilla/js/rhino/src/org/mozilla/javascript/Token.java,v
retrieving revision 1.36
diff -u -r1.36 Token.java
--- src/org/mozilla/javascript/Token.java	10 Nov 2006 15:27:37 -0000	1.36
+++ src/org/mozilla/javascript/Token.java	20 Nov 2006 10:32:24 -0000
@@ -41,6 +41,8 @@
 
 package org.mozilla.javascript;
 
+import java.io.IOException;
+
 /**
  * This class implements the JavaScript scanner.
  *
@@ -404,4 +406,41 @@
         // Token without name
         throw new IllegalStateException(String.valueOf(token));
     }
+
+    /**
+     * The token type.
+     */
+    public final int type;
+
+    /**
+     * The string value for string tokens.
+     */
+    public final String string;
+
+    /**
+     * The line number of the token.
+     */
+    public final int lineno;
+
+    /**
+     * The line offset of the token.
+     */
+    public final int offset;
+
+    /**
+     * The Token class is usually not instanciated when parsing a JavaScript
+     * resource. This constructor is only used by
+     * {@link org.mozilla.javascript.Parser#parseTokens(java.io.Reader, String, int)}
+     * to get information about lexical tokens.
+     * @param type the token type
+     * @param string the string value for string tokens
+     * @param lineno the line number of the token
+     * @param offset the line offset of the token
+     */
+    public Token(int type, String string, int lineno, int offset) {
+        this.type = type;
+        this.string = string;
+        this.lineno = lineno;
+        this.offset = offset;
+    }
 }