syntax: support setting initial line/col for scanner (#349)

The new FilePortion type, which may be provided to the scanner,
parser, or ExecFile functions, combines a piece of text along
with its start line/column numbers, for applications that
extract a Starlark expression from the middle of a larger file.

Fixes https://github.com/google/starlark-go/issues/346

3 files changed, 38 insertions, 2 deletions

diff --git a/syntax/parse.go b/syntax/parse.go
index 0281e4b..50b8087 100644
--- a/syntax/parse.go
+++ b/syntax/parse.go
@@ -28,7 +28,7 @@ const (
 // If src != nil, ParseFile parses the source from src and the filename
 // is only used when recording position information.
 // The type of the argument for the src parameter must be string,
-// []byte, or io.Reader.
+// []byte, io.Reader, or FilePortion.
 // If src == nil, ParseFile parses the file specified by filename.
 func Parse(filename string, src interface{}, mode Mode) (f *File, err error) {
 	in, err := newScanner(filename, src, mode&RetainComments != 0)
diff --git a/syntax/parse_test.go b/syntax/parse_test.go
index 76f9eb3..6052e79 100644
--- a/syntax/parse_test.go
+++ b/syntax/parse_test.go
@@ -439,6 +439,27 @@ func TestParseErrors(t *testing.T) {
 	}
 }
 
+func TestFilePortion(t *testing.T) {
+	// Imagine that the Starlark file or expression print(x.f) is extracted
+	// from the middle of a file in some hypothetical template language;
+	// see https://github.com/google/starlark-go/issues/346. For example:
+	// --
+	// {{loop x seq}}
+	//   {{print(x.f)}}
+	// {{end}}
+	// --
+	fp := syntax.FilePortion{Content: []byte("print(x.f)"), FirstLine: 2, FirstCol: 4}
+	file, err := syntax.Parse("foo.template", fp, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	span := fmt.Sprint(file.Stmts[0].Span())
+	want := "foo.template:2:4 foo.template:2:14"
+	if span != want {
+		t.Errorf("wrong span: got %q, want %q", span, want)
+	}
+}
+
 // dataFile is the same as starlarktest.DataFile.
 // We make a copy to avoid a dependency cycle.
 var dataFile = func(pkgdir, filename string) string {
diff --git a/syntax/scan.go b/syntax/scan.go
index 53d9f5c..a162264 100644
--- a/syntax/scan.go
+++ b/syntax/scan.go
@@ -182,6 +182,15 @@ var tokenNames = [...]string{
 	WHILE:         "while",
 }
 
+// A FilePortion describes the content of a portion of a file.
+// Callers may provide a FilePortion for the src argument of Parse
+// when the desired initial line and column numbers are not (1, 1),
+// such as when an expression is parsed from within larger file.
+type FilePortion struct {
+	Content             []byte
+	FirstLine, FirstCol int32
+}
+
 // A Position describes the location of a rune of input.
 type Position struct {
 	file *string // filename (indirect for compactness)
@@ -249,8 +258,12 @@ type scanner struct {
 }
 
 func newScanner(filename string, src interface{}, keepComments bool) (*scanner, error) {
+	var firstLine, firstCol int32 = 1, 1
+	if portion, ok := src.(FilePortion); ok {
+		firstLine, firstCol = portion.FirstLine, portion.FirstCol
+	}
 	sc := &scanner{
-		pos:          Position{file: &filename, Line: 1, Col: 1},
+		pos:          MakePosition(&filename, firstLine, firstCol),
 		indentstk:    make([]int, 1, 10), // []int{0} + spare capacity
 		lineStart:    true,
 		keepComments: keepComments,
@@ -279,6 +292,8 @@ func readSource(filename string, src interface{}) ([]byte, error) {
 			return nil, err
 		}
 		return data, nil
+	case FilePortion:
+		return src.Content, nil
 	case nil:
 		return ioutil.ReadFile(filename)
 	default: