library: go/{token, scanner}

README.md

@@ -294,6 +294,7 @@ Here are the Go packages that can be imported correctly:
 * [fmt](https://pkg.go.dev/fmt) (partially)
 * [reflect](https://pkg.go.dev/reflect) (partially)
 * [time](https://pkg.go.dev/time) (partially)
+* [encoding](https://pkg.go.dev/encoding)
 * [encoding/binary](https://pkg.go.dev/encoding/binary)
 * [encoding/hex](https://pkg.go.dev/encoding/hex)
 * [encoding/base32](https://pkg.go.dev/encoding/base32)
@@ -307,6 +308,8 @@ Here are the Go packages that can be imported correctly:
 * [crypto/rand](https://pkg.go.dev/crypto/rand) (partially)
 * [regexp](https://pkg.go.dev/regexp)
 * [regexp/syntax](https://pkg.go.dev/regexp/syntax)
+* [go/token](https://pkg.go.dev/go/token)
+* [go/scanner](https://pkg.go.dev/go/scanner)
 
 
 ## Dependencies

_cmptest/goscandemo/scan.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"fmt"
+	"go/scanner"
+	"go/token"
+)
+
+func main() {
+	// src is the input that we want to tokenize.
+	src := []byte("cos(x) + 1i*sin(x) // Euler")
+
+	// Initialize the scanner.
+	var s scanner.Scanner
+	fset := token.NewFileSet()                      // positions are relative to fset
+	file := fset.AddFile("", fset.Base(), len(src)) // register input "file"
+	s.Init(file, src, nil /* no error handler */, scanner.ScanComments)
+
+	// Repeated calls to Scan yield the token sequence found in the input.
+	for {
+		pos, tok, lit := s.Scan()
+		if tok == token.EOF {
+			break
+		}
+		fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit)
+	}
+}