FreeChainXenon/gcc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

libgo: update to Go1.14beta1

Browse source 
Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/214297
This commit is contained in:
Ian Lance Taylor 2020-01-02 15:05:27 -08:00
parent 6ac6529e15
commit 5a8ea16592
1231 changed files with 67372 additions and 21287 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
gcc/go/gofrontend/MERGE
View file

@ -1,4 +1,4 @@
a69ad9c7d1b45edcf8062a07d3a3c9b6838c04f8
c2225a76d1e15f28056596807ebbbc526d4c58da
The first line of this file holds the git revision number of the last
merge done from the gofrontend repository.

6
gotools/ChangeLog
View file

@ -1,3 +1,9 @@
2020-01-21 Ian Lance Taylor <iant@golang.org>
* Makefile.am (gofmt$(EXEEXT)): Link against $(LIBGOTOOL).
(check-go-tool): Copy some vendor directories.
* Makefile.in: Regenerate.
2019-09-06 Ian Lance Taylor <iant@golang.org>
* Makefile.am (check-carchive-test): Just run "go test", not "go

6
gotools/Makefile.am
View file

@ -116,7 +116,7 @@ man_MANS = go.1 gofmt.1
go$(EXEEXT): $(go_cmd_go_files) $(LIBGOTOOL) $(LIBGODEP)
$(GOLINK) $(go_cmd_go_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
gofmt$(EXEEXT): $(go_cmd_gofmt_files) $(LIBGODEP)
$(GOLINK) $(go_cmd_gofmt_files) $(LIBS) $(NET_LIBS)
$(GOLINK) $(go_cmd_gofmt_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBGODEP)
$(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
vet$(EXEEXT): $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBGODEP)
@ -215,6 +215,10 @@ check-go-tool: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
cp $(libgodir)/zdefaultcc.go check-go-dir/src/cmd/go/internal/cfg/
cp -r $(cmdsrcdir)/go/testdata check-go-dir/src/cmd/go/
cp -r $(cmdsrcdir)/internal check-go-dir/src/cmd/
$(MKDIR_P) check-go-dir/src/cmd/vendor/golang.org/x
cp -r $(libgosrcdir)/golang.org/x/mod check-go-dir/src/cmd/vendor/golang.org/x/
cp -r $(libgosrcdir)/golang.org/x/crypto check-go-dir/src/cmd/vendor/golang.org/x/
cp -r $(libgosrcdir)/golang.org/x/xerrors check-go-dir/src/cmd/vendor/golang.org/x/
cp $(libgodir)/objabi.go check-go-dir/src/cmd/internal/objabi/
@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
abs_checkdir=`cd check-go-dir && $(PWD_COMMAND)`; \

6
gotools/Makefile.in
View file

@ -820,7 +820,7 @@ mostlyclean-local:
@NATIVE_TRUE@go$(EXEEXT): $(go_cmd_go_files) $(LIBGOTOOL) $(LIBGODEP)
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_go_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
@NATIVE_TRUE@gofmt$(EXEEXT): $(go_cmd_gofmt_files) $(LIBGODEP)
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_gofmt_files) $(LIBS) $(NET_LIBS)
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_gofmt_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
@NATIVE_TRUE@cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBGODEP)
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
@NATIVE_TRUE@vet$(EXEEXT): $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBGODEP)
@ -886,6 +886,10 @@ mostlyclean-local:
@NATIVE_TRUE@ cp $(libgodir)/zdefaultcc.go check-go-dir/src/cmd/go/internal/cfg/
@NATIVE_TRUE@ cp -r $(cmdsrcdir)/go/testdata check-go-dir/src/cmd/go/
@NATIVE_TRUE@ cp -r $(cmdsrcdir)/internal check-go-dir/src/cmd/
@NATIVE_TRUE@ $(MKDIR_P) check-go-dir/src/cmd/vendor/golang.org/x
@NATIVE_TRUE@ cp -r $(libgosrcdir)/golang.org/x/mod check-go-dir/src/cmd/vendor/golang.org/x/
@NATIVE_TRUE@ cp -r $(libgosrcdir)/golang.org/x/crypto check-go-dir/src/cmd/vendor/golang.org/x/
@NATIVE_TRUE@ cp -r $(libgosrcdir)/golang.org/x/xerrors check-go-dir/src/cmd/vendor/golang.org/x/
@NATIVE_TRUE@ cp $(libgodir)/objabi.go check-go-dir/src/cmd/internal/objabi/
@NATIVE_TRUE@ @abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
@NATIVE_TRUE@ abs_checkdir=`cd check-go-dir && $(PWD_COMMAND)`; \

2
libgo/MERGE
View file

@ -1,4 +1,4 @@
cc8838d645b2b7026c1f3aaceb011775c5ca3a08
a5bfd9da1d1b24f326399b6b75558ded14514f23
The first line of this file holds the git revision number of the
last merge done from the master library sources.

4
libgo/Makefile.am
View file

@ -262,7 +262,8 @@ toolexeclibgohash_DATA = \
hash/adler32.gox \
hash/crc32.gox \
hash/crc64.gox \
hash/fnv.gox
hash/fnv.gox \
hash/maphash.gox
toolexeclibgohtmldir = $(toolexeclibgodir)/html
@ -402,6 +403,7 @@ toolexeclibgounicode_DATA = \
noinst_DATA = \
golang.org/x/net/nettest.gox \
internal/cfg.gox \
internal/obscuretestdata.gox \
internal/testenv.gox \
internal/trace.gox \
net/internal/socktest.gox \

10
libgo/Makefile.in
View file

@ -743,7 +743,8 @@ toolexeclibgohash_DATA = \
hash/adler32.gox \
hash/crc32.gox \
hash/crc64.gox \
hash/fnv.gox
hash/fnv.gox \
hash/maphash.gox
toolexeclibgohtmldir = $(toolexeclibgodir)/html
toolexeclibgohtml_DATA = \
@ -861,9 +862,10 @@ toolexeclibgounicode_DATA = \
# internal packages nothing will explicitly depend on them.
# Force them to be built.
noinst_DATA = golang.org/x/net/nettest.gox internal/cfg.gox \
internal/testenv.gox internal/trace.gox \
net/internal/socktest.gox os/signal/internal/pty.gox \
runtime/pprof/internal/profile.gox zdefaultcc.go
internal/obscuretestdata.gox internal/testenv.gox \
internal/trace.gox net/internal/socktest.gox \
os/signal/internal/pty.gox runtime/pprof/internal/profile.gox \
zdefaultcc.go
@LIBGO_IS_RTEMS_FALSE@rtems_task_variable_add_file =
@LIBGO_IS_RTEMS_TRUE@rtems_task_variable_add_file = runtime/rtems-task-variable-add.c
runtime_context_asm_file = $(am__append_3)

2
libgo/VERSION
View file

@ -1 +1 @@
go1.13
go1.14beta1

7
libgo/check-packages.txt
View file

@ -3,7 +3,6 @@ archive/zip
bufio
bytes
cmd/go/internal/cache
cmd/go/internal/dirhash
cmd/go/internal/generate
cmd/go/internal/get
cmd/go/internal/imports
@ -13,13 +12,10 @@ cmd/go/internal/lockedfile/internal/filelock
cmd/go/internal/modconv
cmd/go/internal/modfetch
cmd/go/internal/modfetch/codehost
cmd/go/internal/modfile
cmd/go/internal/modload
cmd/go/internal/module
cmd/go/internal/mvs
cmd/go/internal/par
cmd/go/internal/search
cmd/go/internal/semver
cmd/go/internal/txtar
cmd/go/internal/work
cmd/internal/buildid
@ -96,6 +92,7 @@ hash/adler32
hash/crc32
hash/crc64
hash/fnv
hash/maphash
html
html/template
image
@ -163,6 +160,8 @@ strings
sync
sync/atomic
syscall
testing
testing/iotest
testing/quick
text/scanner
text/tabwriter

2
libgo/configure vendored
View file

@ -2544,7 +2544,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
ac_config_headers="$ac_config_headers config.h"
libtool_VERSION=15:0:0
libtool_VERSION=16:0:0
# Default to --enable-multilib

2
libgo/configure.ac
View file

@ -10,7 +10,7 @@ AC_INIT(package-unused, version-unused,, libgo)
AC_CONFIG_SRCDIR(Makefile.am)
AC_CONFIG_HEADER(config.h)
libtool_VERSION=15:0:0
libtool_VERSION=16:0:0
AC_SUBST(libtool_VERSION)
AM_ENABLE_MULTILIB(, ..)

2
libgo/go/archive/tar/reader.go
View file

@ -433,7 +433,7 @@ func (tr *Reader) readHeader() (*Header, *block, error) {
// files generated by a pre-Go1.8 toolchain. If the generated file
// happened to have a prefix field that parses as valid
// atime and ctime fields (e.g., when they are valid octal strings),
// then it is impossible to distinguish between an valid GNU file
// then it is impossible to distinguish between a valid GNU file
// and an invalid pre-Go1.8 file.
//
// See https://golang.org/issues/12594

4
libgo/go/archive/zip/reader.go
View file

@ -8,7 +8,6 @@ import (
"bufio"
"encoding/binary"
"errors"
"fmt"
"hash"
"hash/crc32"
"io"
@ -84,9 +83,6 @@ func (z *Reader) init(r io.ReaderAt, size int64) error {
if err != nil {
return err
}
if end.directoryRecords > uint64(size)/fileHeaderLen {
return fmt.Errorf("archive/zip: TOC declares impossible %d files in %d byte zip", end.directoryRecords, size)
}
z.r = r
z.File = make([]*File, 0, end.directoryRecords)
z.Comment = end.comment

46
libgo/go/archive/zip/reader_test.go
View file

@ -8,6 +8,7 @@ import (
"bytes"
"encoding/binary"
"encoding/hex"
"internal/obscuretestdata"
"io"
"io/ioutil"
"os"
@ -19,11 +20,12 @@ import (
)
type ZipTest struct {
Name string
Source func() (r io.ReaderAt, size int64) // if non-nil, used instead of testdata/<Name> file
Comment string
File []ZipTestFile
Error error // the error that Opening this file should return
Name string
Source func() (r io.ReaderAt, size int64) // if non-nil, used instead of testdata/<Name> file
Comment string
File []ZipTestFile
Obscured bool // needed for Apple notarization (golang.org/issue/34986)
Error error // the error that Opening this file should return
}
type ZipTestFile struct {
@ -189,8 +191,12 @@ var tests = []ZipTest{
},
{
// created by Go, before we wrote the "optional" data
// descriptor signatures (which are required by OS X)
Name: "go-no-datadesc-sig.zip",
// descriptor signatures (which are required by macOS).
// Use obscured file to avoid Apple’s notarization service
// rejecting the toolchain due to an inability to unzip this archive.
// See golang.org/issue/34986
Name: "go-no-datadesc-sig.zip.base64",
Obscured: true,
File: []ZipTestFile{
{
Name: "foo.txt",
@ -208,7 +214,7 @@ var tests = []ZipTest{
},
{
// created by Go, after we wrote the "optional" data
// descriptor signatures (which are required by OS X)
// descriptor signatures (which are required by macOS)
Name: "go-with-datadesc-sig.zip",
File: []ZipTestFile{
{
@ -496,8 +502,18 @@ func readTestZip(t *testing.T, zt ZipTest) {
rat, size := zt.Source()
z, err = NewReader(rat, size)
} else {
path := filepath.Join("testdata", zt.Name)
if zt.Obscured {
tf, err := obscuretestdata.DecodeToTempFile(path)
if err != nil {
t.Errorf("obscuretestdata.DecodeToTempFile(%s): %v", path, err)
return
}
defer os.Remove(tf)
path = tf
}
var rc *ReadCloser
rc, err = OpenReader(filepath.Join("testdata", zt.Name))
rc, err = OpenReader(path)
if err == nil {
defer rc.Close()
z = &rc.Reader
@ -981,15 +997,17 @@ func TestIssue10957(t *testing.T) {
}
}
// Verify the number of files is sane.
// Verify that this particular malformed zip file is rejected.
func TestIssue10956(t *testing.T) {
data := []byte("PK\x06\x06PK\x06\a0000\x00\x00\x00\x00\x00\x00\x00\x00" +
"0000PK\x05\x06000000000000" +
"0000\v\x00000\x00\x00\x00\x00\x00\x00\x000")
_, err := NewReader(bytes.NewReader(data), int64(len(data)))
const want = "TOC declares impossible 3472328296227680304 files in 57 byte"
if err == nil && !strings.Contains(err.Error(), want) {
t.Errorf("error = %v; want %q", err, want)
r, err := NewReader(bytes.NewReader(data), int64(len(data)))
if err == nil {
t.Errorf("got nil error, want ErrFormat")
}
if r != nil {
t.Errorf("got non-nil Reader, want nil")
}
}

BIN
libgo/go/archive/zip/testdata/go-no-datadesc-sig.zip vendored
View file

Binary file not shown.

1
libgo/go/archive/zip/testdata/go-no-datadesc-sig.zip.base64 vendored Normal file
View file

@ -0,0 +1 @@
UEsDBBQACAAAAGWHaECoZTJ+BAAAAAQAAAAHABgAZm9vLnR4dFVUBQAD3lVZT3V4CwABBPUBAAAEFAAAAGZvbwqoZTJ+BAAAAAQAAABQSwMEFAAIAAAAZodoQOmzogQEAAAABAAAAAcAGABiYXIudHh0VVQFAAPgVVlPdXgLAAEE9QEAAAQUAAAAYmFyCumzogQEAAAABAAAAFBLAQIUAxQACAAAAGWHaECoZTJ+BAAAAAQAAAAHABgAAAAAAAAAAACkgQAAAABmb28udHh0VVQFAAPeVVlPdXgLAAEE9QEAAAQUAAAAUEsBAhQDFAAIAAAAZodoQOmzogQEAAAABAAAAAcAGAAAAAAAAAAAAKSBTQAAAGJhci50eHRVVAUAA+BVWU91eAsAAQT1AQAABBQAAABQSwUGAAAAAAIAAgCaAAAAmgAAAAAA

17
libgo/go/bufio/bufio.go
View file

@ -432,6 +432,7 @@ func (b *Reader) ReadBytes(delim byte) ([]byte, error) {
var frag []byte
var full [][]byte
var err error
n := 0
for {
var e error
frag, e = b.ReadSlice(delim)
@ -447,18 +448,15 @@ func (b *Reader) ReadBytes(delim byte) ([]byte, error) {
buf := make([]byte, len(frag))
copy(buf, frag)
full = append(full, buf)
n += len(buf)
}
// Allocate new buffer to hold the full pieces and the fragment.
n := 0
for i := range full {
n += len(full[i])
}
n += len(frag)
// Copy full pieces and fragment in.
// Allocate new buffer to hold the full pieces and the fragment.
buf := make([]byte, n)
n = 0
// Copy full pieces and fragment in.
for i := range full {
n += copy(buf[n:], full[i])
}
@ -708,9 +706,14 @@ func (b *Writer) WriteString(s string) (int, error) {
// supports the ReadFrom method, and b has no buffered data yet,
// this calls the underlying ReadFrom without buffering.
func (b *Writer) ReadFrom(r io.Reader) (n int64, err error) {
if b.err != nil {
return 0, b.err
}
if b.Buffered() == 0 {
if w, ok := b.wr.(io.ReaderFrom); ok {
return w.ReadFrom(r)
n, err = w.ReadFrom(r)
b.err = err
return n, err
}
}
var m int

46
libgo/go/bufio/bufio_test.go
View file

@ -1535,6 +1535,52 @@ func TestPartialReadEOF(t *testing.T) {
}
}
type writerWithReadFromError struct{}
func (w writerWithReadFromError) ReadFrom(r io.Reader) (int64, error) {
return 0, errors.New("writerWithReadFromError error")
}
func (w writerWithReadFromError) Write(b []byte) (n int, err error) {
return 10, nil
}
func TestWriterReadFromMustSetUnderlyingError(t *testing.T) {
var wr = NewWriter(writerWithReadFromError{})
if _, err := wr.ReadFrom(strings.NewReader("test2")); err == nil {
t.Fatal("expected ReadFrom returns error, got nil")
}
if _, err := wr.Write([]byte("123")); err == nil {
t.Fatal("expected Write returns error, got nil")
}
}
type writeErrorOnlyWriter struct{}
func (w writeErrorOnlyWriter) Write(p []byte) (n int, err error) {
return 0, errors.New("writeErrorOnlyWriter error")
}
// Ensure that previous Write errors are immediately returned
// on any ReadFrom. See golang.org/issue/35194.
func TestWriterReadFromMustReturnUnderlyingError(t *testing.T) {
var wr = NewWriter(writeErrorOnlyWriter{})
s := "test1"
wantBuffered := len(s)
if _, err := wr.WriteString(s); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := wr.Flush(); err == nil {
t.Error("expected flush error, got nil")
}
if _, err := wr.ReadFrom(strings.NewReader("test2")); err == nil {
t.Fatal("expected error, got nil")
}
if buffered := wr.Buffered(); buffered != wantBuffered {
t.Fatalf("Buffered = %v; want %v", buffered, wantBuffered)
}
}
func BenchmarkReaderCopyOptimal(b *testing.B) {
// Optimal case is where the underlying reader implements io.WriterTo
srcBuf := bytes.NewBuffer(make([]byte, 8192))

3
libgo/go/bytes/bytes.go
View file

@ -935,7 +935,8 @@ func ReplaceAll(s, old, new []byte) []byte {
}
// EqualFold reports whether s and t, interpreted as UTF-8 strings,
// are equal under Unicode case-folding.
// are equal under Unicode case-folding, which is a more general
// form of case-insensitivity.
func EqualFold(s, t []byte) bool {
for len(s) != 0 && len(t) != 0 {
// Extract first rune from each.

33
libgo/go/bytes/compare_test.go
View file

@ -120,6 +120,39 @@ func TestCompareBytes(t *testing.T) {
}
}
func TestEndianBaseCompare(t *testing.T) {
// This test compares byte slices that are almost identical, except one
// difference that for some j, a[j]>b[j] and a[j+1]<b[j+1]. If the implementation
// compares large chunks with wrong endianness, it gets wrong result.
// no vector register is larger than 512 bytes for now
const maxLength = 512
a := make([]byte, maxLength)
b := make([]byte, maxLength)
// randomish but deterministic data. No 0 or 255.
for i := 0; i < maxLength; i++ {
a[i] = byte(1 + 31*i%254)
b[i] = byte(1 + 31*i%254)
}
for i := 2; i <= maxLength; i <<= 1 {
for j := 0; j < i-1; j++ {
a[j] = b[j] - 1
a[j+1] = b[j+1] + 1
cmp := Compare(a[:i], b[:i])
if cmp != -1 {
t.Errorf(`CompareBbigger(%d,%d) = %d`, i, j, cmp)
}
a[j] = b[j] + 1
a[j+1] = b[j+1] - 1
cmp = Compare(a[:i], b[:i])
if cmp != 1 {
t.Errorf(`CompareAbigger(%d,%d) = %d`, i, j, cmp)
}
a[j] = b[j]
a[j+1] = b[j+1]
}
}
}
func BenchmarkCompareBytesEqual(b *testing.B) {
b1 := []byte("Hello Gophers!")
b2 := []byte("Hello Gophers!")

4
libgo/go/cmd/cgo/doc.go
View file

@ -55,7 +55,7 @@ For example:
The default pkg-config tool may be changed by setting the PKG_CONFIG environment variable.
For security reasons, only a limited set of flags are allowed, notably -D, -I, and -l.
For security reasons, only a limited set of flags are allowed, notably -D, -U, -I, and -l.
To allow additional flags, set CGO_CFLAGS_ALLOW to a regular expression
matching the new flags. To disallow flags that would otherwise be allowed,
set CGO_CFLAGS_DISALLOW to a regular expression matching arguments
@ -99,7 +99,7 @@ Will be expanded to:
When the Go tool sees that one or more Go files use the special import
"C", it will look for other non-Go files in the directory and compile
them as part of the Go package. Any .c, .s, or .S files will be
them as part of the Go package. Any .c, .s, .S or .sx files will be
compiled with the C compiler. Any .cc, .cpp, or .cxx files will be
compiled with the C++ compiler. Any .f, .F, .for or .f90 files will be
compiled with the fortran compiler. Any .h, .hh, .hpp, or .hxx files will

21
libgo/go/cmd/cgo/gcc.go
View file

@ -834,7 +834,7 @@ func (p *Package) rewriteCall(f *File, call *Call) (string, bool) {
// Rewrite C.f(p) to
// func() {
// _cgo0 := p
// _cgoCheckPointer(_cgo0)
// _cgoCheckPointer(_cgo0, nil)
// C.f(_cgo0)
// }()
// Using a function literal like this lets us evaluate the
@ -852,7 +852,7 @@ func (p *Package) rewriteCall(f *File, call *Call) (string, bool) {
// defer func() func() {
// _cgo0 := p
// return func() {
// _cgoCheckPointer(_cgo0)
// _cgoCheckPointer(_cgo0, nil)
// C.f(_cgo0)
// }
// }()()
@ -939,7 +939,7 @@ func (p *Package) rewriteCall(f *File, call *Call) (string, bool) {
}
fmt.Fprintf(&sb, "_cgo%d := %s; ", i, gofmtPos(arg, origArg.Pos()))
fmt.Fprintf(&sbCheck, "_cgoCheckPointer(_cgo%d); ", i)
fmt.Fprintf(&sbCheck, "_cgoCheckPointer(_cgo%d, nil); ", i)
}
if call.Deferred {
@ -2211,6 +2211,11 @@ func (c *typeConv) FinishType(pos token.Pos) {
// Type returns a *Type with the same memory layout as
// dtype when used as the type of a variable or a struct field.
func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
return c.loadType(dtype, pos, "")
}
// loadType recursively loads the requested dtype and its dependency graph.
func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Type {
// Always recompute bad pointer typedefs, as the set of such
// typedefs changes as we see more types.
checkCache := true
@ -2218,7 +2223,9 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
checkCache = false
}
key := dtype.String()
// The cache key should be relative to its parent.
// See issue https://golang.org/issue/31891
key := parent + " > " + dtype.String()
if checkCache {
if t, ok := c.m[key]; ok {
@ -2258,7 +2265,7 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
// Translate to zero-length array instead.
count = 0
}
sub := c.Type(dt.Type, pos)
sub := c.loadType(dt.Type, pos, key)
t.Align = sub.Align
t.Go = &ast.ArrayType{
Len: c.intExpr(count),
@ -2403,7 +2410,7 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
c.ptrs[key] = append(c.ptrs[key], t)
case *dwarf.QualType:
t1 := c.Type(dt.Type, pos)
t1 := c.loadType(dt.Type, pos, key)
t.Size = t1.Size
t.Align = t1.Align
t.Go = t1.Go
@ -2487,7 +2494,7 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
}
name := c.Ident("_Ctype_" + dt.Name)
goIdent[name.Name] = name
sub := c.Type(dt.Type, pos)
sub := c.loadType(dt.Type, pos, key)
if c.badPointerTypedef(dt) {
// Treat this typedef as a uintptr.
s := *sub

12
libgo/go/cmd/cgo/out.go
View file

@ -1638,14 +1638,14 @@ func _cgo_runtime_cgocall(unsafe.Pointer, uintptr) int32
func _cgo_runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr, uintptr)
//go:linkname _cgoCheckPointer runtime.cgoCheckPointer
func _cgoCheckPointer(interface{}, ...interface{})
func _cgoCheckPointer(interface{}, interface{})
//go:linkname _cgoCheckResult runtime.cgoCheckResult
func _cgoCheckResult(interface{})
`
const gccgoGoProlog = `
func _cgoCheckPointer(interface{}, ...interface{})
func _cgoCheckPointer(interface{}, interface{})
func _cgoCheckResult(interface{})
`
@ -1832,16 +1832,16 @@ typedef struct __go_empty_interface {
void *__object;
} Eface;
extern void runtimeCgoCheckPointer(Eface, Slice)
extern void runtimeCgoCheckPointer(Eface, Eface)
__asm__("runtime.cgoCheckPointer")
__attribute__((weak));
extern void localCgoCheckPointer(Eface, Slice)
extern void localCgoCheckPointer(Eface, Eface)
__asm__("GCCGOSYMBOLPREF._cgoCheckPointer");
void localCgoCheckPointer(Eface ptr, Slice args) {
void localCgoCheckPointer(Eface ptr, Eface arg) {
if(runtimeCgoCheckPointer) {
runtimeCgoCheckPointer(ptr, args);
runtimeCgoCheckPointer(ptr, arg);
}
}

87
libgo/go/cmd/go/alldocs.go
View file

@ -110,11 +110,13 @@
// The default is the number of CPUs available.
// -race
// enable data race detection.
// Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64.
// Supported only on linux/amd64, freebsd/amd64, darwin/amd64, windows/amd64,
// linux/ppc64le and linux/arm64 (only for 48-bit VMA).
// -msan
// enable interoperation with memory sanitizer.
// Supported only on linux/amd64, linux/arm64
// and only with Clang/LLVM as the host C compiler.
// On linux/arm64, pie build mode will be used.
// -v
// print the names of packages as they are compiled.
// -work
@ -143,11 +145,21 @@
// -ldflags '[pattern=]arg list'
// arguments to pass on each go tool link invocation.
// -linkshared
// link against shared libraries previously created with
// -buildmode=shared.
// build code that will be linked against shared libraries previously
// created with -buildmode=shared.
// -mod mode
// module download mode to use: readonly or vendor.
// module download mode to use: readonly, vendor, or mod.
// See 'go help modules' for more.
// -modcacherw
// leave newly-created directories in the module cache read-write
// instead of making them read-only.
// -modfile file
// in module aware mode, read (and possibly write) an alternate go.mod
// file instead of the one in the module root directory. A file named
// "go.mod" must still be present in order to determine the module root
// directory, but it is not accessed. When -modfile is specified, an
// alternate go.sum file is also used: its path is derived from the
// -modfile flag by trimming the ".mod" extension and appending ".sum".
// -pkgdir dir
// install and load all packages from dir instead of the usual locations.
// For example, when building with a non-standard configuration,
@ -361,6 +373,8 @@
// Treat a command (package main) like a regular package.
// Otherwise package main's exported symbols are hidden
// when showing the package's top-level documentation.
// -short
// One-line representation for each symbol.
// -src
// Show the full source code for the symbol. This will
// display the full Go source of its declaration and
@ -431,6 +445,9 @@
// The -n flag prints commands that would be executed.
// The -x flag prints commands as they are executed.
//
// The -mod flag's value sets which module download mode
// to use: readonly or vendor. See 'go help modules' for more.
//
// To run gofmt with specific options, run gofmt itself.
//
// See also: go fix, go vet.
@ -993,7 +1010,7 @@
//
// Usage:
//
// go mod download [-json] [modules]
// go mod download [-x] [-json] [modules]
//
// Download downloads the named modules, which can be module patterns selecting
// dependencies of the main module or module queries of the form path@version.
@ -1018,9 +1035,10 @@
// Dir string // absolute path to cached source root directory
// Sum string // checksum for path, version (as in go.sum)
// GoModSum string // checksum for go.mod (as in go.sum)
// Latest bool // would @latest resolve to this version?
// }
//
// The -x flag causes download to print the commands download executes.
//
// See 'go help modules' for more about module queries.
//
//
@ -1057,12 +1075,17 @@
// add and drop an exclusion for the given module path and version.
// Note that -exclude=path@version is a no-op if that exclusion already exists.
//
// The -replace=old[@v]=new[@v] and -dropreplace=old[@v] flags
// add and drop a replacement of the given module path and version pair.
// If the @v in old@v is omitted, the replacement applies to all versions
// with the old module path. If the @v in new@v is omitted, the new path
// should be a local module root directory, not a module path.
// Note that -replace overrides any existing replacements for old[@v].
// The -replace=old[@v]=new[@v] flag adds a replacement of the given
// module path and version pair. If the @v in old@v is omitted, a
// replacement without a version on the left side is added, which applies
// to all versions of the old module path. If the @v in new@v is omitted,
// the new path should be a local module root directory, not a module
// path. Note that -replace overrides any redundant replacements for old[@v],
// so omitting @v will drop existing replacements for specific versions.
//
// The -dropreplace=old[@v] flag drops a replacement of the given
// module path and version pair. If the @v is omitted, a replacement without
// a version on the left side is dropped.
//
// The -require, -droprequire, -exclude, -dropexclude, -replace,
// and -dropreplace editing flags may be repeated, and the changes
@ -1232,7 +1255,7 @@
// If the -exec flag is not given, GOOS or GOARCH is different from the system
// default, and a program named go_$GOOS_$GOARCH_exec can be found
// on the current search path, 'go run' invokes the binary using that program,
// for example 'go_nacl_386_exec a.out arguments...'. This allows execution of
// for example 'go_js_wasm_exec a.out arguments...'. This allows execution of
// cross-compiled programs when a simulator or other execution method is
// available.
//
@ -1504,8 +1527,8 @@
// extension will be passed to SWIG. Any file with a .swigcxx extension
// will be passed to SWIG with the -c++ option.
//
// When either cgo or SWIG is used, go build will pass any .c, .m, .s,
// or .S files to the C compiler, and any .cc, .cpp, .cxx files to the C++
// When either cgo or SWIG is used, go build will pass any .c, .m, .s, .S
// or .sx files to the C compiler, and any .cc, .cpp, .cxx files to the C++
// compiler. The CC or CXX environment variables may be set to determine
// the C or C++ compiler, respectively, to use.
//
@ -1588,6 +1611,10 @@
// Because the entries are space-separated, flag values must
// not contain spaces. Flags listed on the command line
// are applied after this list and therefore override it.
// GOINSECURE
// Comma-separated list of glob patterns (in the syntax of Go's path.Match)
// of module path prefixes that should always be fetched in an insecure
// manner. Only applies to dependencies that are being fetched directly.
// GOOS
// The operating system for which to compile code.
// Examples are linux, darwin, windows, netbsd.
@ -1699,8 +1726,10 @@
// GOHOSTOS
// The operating system (GOOS) of the Go toolchain binaries.
// GOMOD
// The absolute path to the go.mod of the main module,
// or the empty string if not using modules.
// The absolute path to the go.mod of the main module.
// If module-aware mode is enabled, but there is no go.mod, GOMOD will be
// os.DevNull ("/dev/null" on Unix-like systems, "NUL" on Windows).
// If module-aware mode is disabled, GOMOD will be the empty string.
// GOTOOLDIR
// The directory where the go tools (compile, cover, doc, etc...) are installed.
//
@ -1724,7 +1753,7 @@
// .m
// Objective-C source files. Only useful with cgo, and always
// compiled with the OS-native compiler.
// .s, .S
// .s, .S, .sx
// Assembler source files.
// If the package uses cgo or SWIG, these will be assembled with the
// OS-native assembler (typically gcc (sic)); otherwise they
@ -2060,8 +2089,8 @@
//
// The GET requests sent to a Go module proxy are:
//
// GET $GOPROXY/<module>/@v/list returns a list of all known versions of the
// given module, one per line.
// GET $GOPROXY/<module>/@v/list returns a list of known versions of the given
// module, one per line.
//
// GET $GOPROXY/<module>/@v/<version>.info returns JSON-formatted metadata
// about that version of the given module.
@ -2072,6 +2101,21 @@
// GET $GOPROXY/<module>/@v/<version>.zip returns the zip archive
// for that version of the given module.
//
// GET $GOPROXY/<module>/@latest returns JSON-formatted metadata about the
// latest known version of the given module in the same format as
// <module>/@v/<version>.info. The latest version should be the version of
// the module the go command may use if <module>/@v/list is empty or no
// listed version is suitable. <module>/@latest is optional and may not
// be implemented by a module proxy.
//
// When resolving the latest version of a module, the go command will request
// <module>/@v/list, then, if no suitable versions are found, <module>/@latest.
// The go command prefers, in order: the semantically highest release version,
// the semantically highest pre-release version, and the chronologically
// most recent pseudo-version. In Go 1.12 and earlier, the go command considered
// pseudo-versions in <module>/@v/list to be pre-release versions, but this is
// no longer true since Go 1.13.
//
// To avoid problems when serving from case-sensitive file systems,
// the <module> and <version> elements are case-encoded, replacing every
// uppercase letter with an exclamation mark followed by the corresponding
@ -2460,6 +2504,9 @@
// directory holds the correct copies of dependencies and ignores
// the dependency descriptions in go.mod.
//
// If invoked with -mod=mod, the go command loads modules from the module cache
// even if there is a vendor directory present.
//
// Pseudo-versions
//
// The go.mod file and the go command more generally use semantic versions as

905
libgo/go/cmd/go/go_test.go
View file

File diff suppressed because it is too large Load diff

2
libgo/go/cmd/go/help_test.go
View file

@ -2,8 +2,6 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !nacl
package main_test
import (

5
libgo/go/cmd/go/internal/base/flag.go
View file

@ -33,3 +33,8 @@ func AddBuildFlagsNX(flags *flag.FlagSet) {
flags.BoolVar(&cfg.BuildN, "n", false, "")
flags.BoolVar(&cfg.BuildX, "x", false, "")
}
// AddLoadFlags adds the -mod build flag to the flag set.
func AddLoadFlags(flags *flag.FlagSet) {
flags.StringVar(&cfg.BuildMod, "mod", "", "")
}

2
libgo/go/cmd/go/internal/base/signal_unix.go
View file

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build aix darwin dragonfly freebsd hurd js linux nacl netbsd openbsd solaris
// +build aix darwin dragonfly freebsd hurd js linux netbsd openbsd solaris
package base

69
libgo/go/cmd/go/internal/cache/cache.go vendored
View file

@ -74,7 +74,22 @@ func (c *Cache) fileName(id [HashSize]byte, key string) string {
return filepath.Join(c.dir, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+"-"+key)
}
var errMissing = errors.New("cache entry not found")
// An entryNotFoundError indicates that a cache entry was not found, with an
// optional underlying reason.
type entryNotFoundError struct {
Err error
}
func (e *entryNotFoundError) Error() string {
if e.Err == nil {
return "cache entry not found"
}
return fmt.Sprintf("cache entry not found: %v", e.Err)
}
func (e *entryNotFoundError) Unwrap() error {
return e.Err
}
const (
// action entry file is "v1 <hex id> <hex out> <decimal size space-padded to 20 bytes> <unixnano space-padded to 20 bytes>\n"
@ -93,6 +108,8 @@ const (
// GODEBUG=gocacheverify=1.
var verify = false
var errVerifyMode = errors.New("gocachverify=1")
// DebugTest is set when GODEBUG=gocachetest=1 is in the environment.
var DebugTest = false
@ -121,7 +138,7 @@ func initEnv() {
// saved file for that output ID is still available.
func (c *Cache) Get(id ActionID) (Entry, error) {
if verify {
return Entry{}, errMissing
return Entry{}, &entryNotFoundError{Err: errVerifyMode}
}
return c.get(id)
}
@ -134,47 +151,60 @@ type Entry struct {
// get is Get but does not respect verify mode, so that Put can use it.
func (c *Cache) get(id ActionID) (Entry, error) {
missing := func() (Entry, error) {
return Entry{}, errMissing
missing := func(reason error) (Entry, error) {
return Entry{}, &entryNotFoundError{Err: reason}
}
f, err := os.Open(c.fileName(id, "a"))
if err != nil {
return missing()
return missing(err)
}
defer f.Close()
entry := make([]byte, entrySize+1) // +1 to detect whether f is too long
if n, err := io.ReadFull(f, entry); n != entrySize || err != io.ErrUnexpectedEOF {
return missing()
if n, err := io.ReadFull(f, entry); n > entrySize {
return missing(errors.New("too long"))
} else if err != io.ErrUnexpectedEOF {
if err == io.EOF {
return missing(errors.New("file is empty"))
}
return missing(err)
} else if n < entrySize {
return missing(errors.New("entry file incomplete"))
}
if entry[0] != 'v' || entry[1] != '1' || entry[2] != ' ' || entry[3+hexSize] != ' ' || entry[3+hexSize+1+hexSize] != ' ' || entry[3+hexSize+1+hexSize+1+20] != ' ' || entry[entrySize-1] != '\n' {
return missing()
return missing(errors.New("invalid header"))
}
eid, entry := entry[3:3+hexSize], entry[3+hexSize:]
eout, entry := entry[1:1+hexSize], entry[1+hexSize:]
esize, entry := entry[1:1+20], entry[1+20:]
etime, entry := entry[1:1+20], entry[1+20:]
var buf [HashSize]byte
if _, err := hex.Decode(buf[:], eid); err != nil || buf != id {
return missing()
if _, err := hex.Decode(buf[:], eid); err != nil {
return missing(fmt.Errorf("decoding ID: %v", err))
} else if buf != id {
return missing(errors.New("mismatched ID"))
}
if _, err := hex.Decode(buf[:], eout); err != nil {
return missing()
return missing(fmt.Errorf("decoding output ID: %v", err))
}
i := 0
for i < len(esize) && esize[i] == ' ' {
i++
}
size, err := strconv.ParseInt(string(esize[i:]), 10, 64)
if err != nil || size < 0 {
return missing()
if err != nil {
return missing(fmt.Errorf("parsing size: %v", err))
} else if size < 0 {
return missing(errors.New("negative size"))
}
i = 0
for i < len(etime) && etime[i] == ' ' {
i++
}
tm, err := strconv.ParseInt(string(etime[i:]), 10, 64)
if err != nil || tm < 0 {
return missing()
if err != nil {
return missing(fmt.Errorf("parsing timestamp: %v", err))
} else if tm < 0 {
return missing(errors.New("negative timestamp"))
}
c.used(c.fileName(id, "a"))
@ -191,8 +221,11 @@ func (c *Cache) GetFile(id ActionID) (file string, entry Entry, err error) {
}
file = c.OutputFile(entry.OutputID)
info, err := os.Stat(file)
if err != nil || info.Size() != entry.Size {
return "", Entry{}, errMissing
if err != nil {
return "", Entry{}, &entryNotFoundError{Err: err}
}
if info.Size() != entry.Size {
return "", Entry{}, &entryNotFoundError{Err: errors.New("file incomplete")}
}
return file, entry, nil
}
@ -207,7 +240,7 @@ func (c *Cache) GetBytes(id ActionID) ([]byte, Entry, error) {
}
data, _ := ioutil.ReadFile(c.OutputFile(entry.OutputID))
if sha256.Sum256(data) != entry.OutputID {
return nil, entry, errMissing
return nil, entry, &entryNotFoundError{Err: errors.New("bad checksum")}
}
return data, entry, nil
}

15
libgo/go/cmd/go/internal/cfg/cfg.go
View file

@ -27,6 +27,7 @@ var (
BuildBuildmode string // -buildmode flag
BuildContext = defaultContext()
BuildMod string // -mod flag
BuildModReason string // reason -mod flag is set, if set by default
BuildI bool // -i flag
BuildLinkshared bool // -linkshared flag
BuildMSan bool // -msan flag
@ -44,6 +45,9 @@ var (
BuildWork bool // -work flag
BuildX bool // -x flag
ModCacheRW bool // -modcacherw flag
ModFile string // -modfile flag
CmdName string // "build", "install", "list", "mod tidy", etc.
DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable)
@ -241,11 +245,12 @@ var (
GOPPC64 = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", objabi.GOPPC64))
GOWASM = envOr("GOWASM", fmt.Sprint(objabi.GOWASM))
GOPROXY = envOr("GOPROXY", "https://proxy.golang.org,direct")
GOSUMDB = envOr("GOSUMDB", "sum.golang.org")
GOPRIVATE = Getenv("GOPRIVATE")
GONOPROXY = envOr("GONOPROXY", GOPRIVATE)
GONOSUMDB = envOr("GONOSUMDB", GOPRIVATE)
GOPROXY = envOr("GOPROXY", "https://proxy.golang.org,direct")
GOSUMDB = envOr("GOSUMDB", "sum.golang.org")
GOPRIVATE = Getenv("GOPRIVATE")
GONOPROXY = envOr("GONOPROXY", GOPRIVATE)
GONOSUMDB = envOr("GONOSUMDB", GOPRIVATE)
GOINSECURE = Getenv("GOINSECURE")
)
// GetArchEnv returns the name and setting of the

2
libgo/go/cmd/go/internal/clean/clean.go
View file

@ -102,7 +102,7 @@ func init() {
// mentioned explicitly in the docs but they
// are part of the build flags.
work.AddBuildFlags(CmdClean)
work.AddBuildFlags(CmdClean, work.DefaultBuildFlags)
}
func runClean(cmd *base.Command, args []string) {

135
libgo/go/cmd/go/internal/dirhash/hash_test.go
View file

@ -1,135 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package dirhash
import (
"archive/zip"
"crypto/sha256"
"encoding/base64"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
)
func h(s string) string {
return fmt.Sprintf("%x", sha256.Sum256([]byte(s)))
}
func htop(k string, s string) string {
sum := sha256.Sum256([]byte(s))
return k + ":" + base64.StdEncoding.EncodeToString(sum[:])
}
func TestHash1(t *testing.T) {
files := []string{"xyz", "abc"}
open := func(name string) (io.ReadCloser, error) {
return ioutil.NopCloser(strings.NewReader("data for " + name)), nil
}
want := htop("h1", fmt.Sprintf("%s %s\n%s %s\n", h("data for abc"), "abc", h("data for xyz"), "xyz"))
out, err := Hash1(files, open)
if err != nil {
t.Fatal(err)
}
if out != want {
t.Errorf("Hash1(...) = %s, want %s", out, want)
}
_, err = Hash1([]string{"xyz", "a\nbc"}, open)
if err == nil {
t.Error("Hash1: expected error on newline in filenames")
}
}
func TestHashDir(t *testing.T) {
dir, err := ioutil.TempDir("", "dirhash-test-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
if err := ioutil.WriteFile(filepath.Join(dir, "xyz"), []byte("data for xyz"), 0666); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(dir, "abc"), []byte("data for abc"), 0666); err != nil {
t.Fatal(err)
}
want := htop("h1", fmt.Sprintf("%s %s\n%s %s\n", h("data for abc"), "prefix/abc", h("data for xyz"), "prefix/xyz"))
out, err := HashDir(dir, "prefix", Hash1)
if err != nil {
t.Fatalf("HashDir: %v", err)
}
if out != want {
t.Errorf("HashDir(...) = %s, want %s", out, want)
}
}
func TestHashZip(t *testing.T) {
f, err := ioutil.TempFile("", "dirhash-test-")
if err != nil {
t.Fatal(err)
}
defer os.Remove(f.Name())
defer f.Close()
z := zip.NewWriter(f)
w, err := z.Create("prefix/xyz")
if err != nil {
t.Fatal(err)
}
w.Write([]byte("data for xyz"))
w, err = z.Create("prefix/abc")
if err != nil {
t.Fatal(err)
}
w.Write([]byte("data for abc"))
if err := z.Close(); err != nil {
t.Fatal(err)
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
want := htop("h1", fmt.Sprintf("%s %s\n%s %s\n", h("data for abc"), "prefix/abc", h("data for xyz"), "prefix/xyz"))
out, err := HashZip(f.Name(), Hash1)
if err != nil {
t.Fatalf("HashDir: %v", err)
}
if out != want {
t.Errorf("HashDir(...) = %s, want %s", out, want)
}
}
func TestDirFiles(t *testing.T) {
dir, err := ioutil.TempDir("", "dirfiles-test-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
if err := ioutil.WriteFile(filepath.Join(dir, "xyz"), []byte("data for xyz"), 0666); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(dir, "abc"), []byte("data for abc"), 0666); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(dir, "subdir"), 0777); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(dir, "subdir", "xyz"), []byte("data for subdir xyz"), 0666); err != nil {
t.Fatal(err)
}
prefix := "foo/bar@v2.3.4"
out, err := DirFiles(dir, prefix)
if err != nil {
t.Fatalf("DirFiles: %v", err)
}
for _, file := range out {
if !strings.HasPrefix(file, prefix) {
t.Errorf("Dir file = %s, want prefix %s", file, prefix)
}
}
}

2
libgo/go/cmd/go/internal/doc/doc.go
View file

@ -114,6 +114,8 @@ Flags:
Treat a command (package main) like a regular package.
Otherwise package main's exported symbols are hidden
when showing the package's top-level documentation.
-short
One-line representation for each symbol.
-src
Show the full source code for the symbol. This will
display the full Go source of its declaration and

69
libgo/go/cmd/go/internal/envcmd/env.go
View file

@ -8,6 +8,7 @@ package envcmd
import (
"encoding/json"
"fmt"
"go/build"
"io/ioutil"
"os"
"path/filepath"
@ -75,6 +76,7 @@ func MkEnv() []cfg.EnvVar {
{Name: "GOFLAGS", Value: cfg.Getenv("GOFLAGS")},
{Name: "GOHOSTARCH", Value: runtime.GOARCH},
{Name: "GOHOSTOS", Value: runtime.GOOS},
{Name: "GOINSECURE", Value: cfg.GOINSECURE},
{Name: "GONOPROXY", Value: cfg.GONOPROXY},
{Name: "GONOSUMDB", Value: cfg.GONOSUMDB},
{Name: "GOOS", Value: cfg.Goos},
@ -237,7 +239,7 @@ func runEnv(cmd *base.Command, args []string) {
base.Fatalf("go env -w: arguments must be KEY=VALUE: invalid argument: %s", arg)
}
key, val := arg[:i], arg[i+1:]
if err := checkEnvWrite(key, val, env); err != nil {
if err := checkEnvWrite(key, val); err != nil {
base.Fatalf("go env -w: %v", err)
}
if _, ok := add[key]; ok {
@ -248,6 +250,21 @@ func runEnv(cmd *base.Command, args []string) {
fmt.Fprintf(os.Stderr, "warning: go env -w %s=... does not override conflicting OS environment variable\n", key)
}
}
goos, okGOOS := add["GOOS"]
goarch, okGOARCH := add["GOARCH"]
if okGOOS || okGOARCH {
if !okGOOS {
goos = cfg.Goos
}
if !okGOARCH {
goarch = cfg.Goarch
}
if err := work.CheckGOOSARCHPair(goos, goarch); err != nil {
base.Fatalf("go env -w: %v", err)
}
}
updateEnvFile(add, nil)
return
}
@ -259,11 +276,29 @@ func runEnv(cmd *base.Command, args []string) {
}
del := make(map[string]bool)
for _, arg := range args {
if err := checkEnvWrite(arg, "", env); err != nil {
if err := checkEnvWrite(arg, ""); err != nil {
base.Fatalf("go env -u: %v", err)
}
del[arg] = true
}
if del["GOOS"] || del["GOARCH"] {
goos, goarch := cfg.Goos, cfg.Goarch
if del["GOOS"] {
goos = getOrigEnv("GOOS")
if goos == "" {
goos = build.Default.GOOS
}
}
if del["GOARCH"] {
goarch = getOrigEnv("GOARCH")
if goarch == "" {
goarch = build.Default.GOARCH
}
}
if err := work.CheckGOOSARCHPair(goos, goarch); err != nil {
base.Fatalf("go env -u: %v", err)
}
}
updateEnvFile(nil, del)
return
}
@ -330,7 +365,16 @@ func printEnvAsJSON(env []cfg.EnvVar) {
}
}
func checkEnvWrite(key, val string, env []cfg.EnvVar) error {
func getOrigEnv(key string) string {
for _, v := range cfg.OrigEnv {
if strings.HasPrefix(v, key+"=") {
return strings.TrimPrefix(v, key+"=")
}
}
return ""
}
func checkEnvWrite(key, val string) error {
switch key {
case "GOEXE", "GOGCCFLAGS", "GOHOSTARCH", "GOHOSTOS", "GOMOD", "GOTOOLDIR":
return fmt.Errorf("%s cannot be modified", key)
@ -343,6 +387,25 @@ func checkEnvWrite(key, val string, env []cfg.EnvVar) error {
return fmt.Errorf("unknown go command variable %s", key)
}
// Some variables can only have one of a few valid values. If set to an
// invalid value, the next cmd/go invocation might fail immediately,
// even 'go env -w' itself.
switch key {
case "GO111MODULE":
switch val {
case "", "auto", "on", "off":
default:
return fmt.Errorf("invalid %s value %q", key, val)
}
case "GOPATH":
if strings.HasPrefix(val, "~") {
return fmt.Errorf("GOPATH entry cannot start with shell metacharacter '~': %q", val)
}
if !filepath.IsAbs(val) && val != "" {
return fmt.Errorf("GOPATH entry is relative; must be absolute path: %q", val)
}
}
if !utf8.ValidString(val) {
return fmt.Errorf("invalid UTF-8 in %s=... value", key)
}

6
libgo/go/cmd/go/internal/fmtcmd/fmt.go
View file

@ -22,6 +22,7 @@ import (
func init() {
base.AddBuildFlagsNX(&CmdFmt.Flag)
base.AddLoadFlags(&CmdFmt.Flag)
}
var CmdFmt = &base.Command{
@ -38,6 +39,9 @@ For more about specifying packages, see 'go help packages'.
The -n flag prints commands that would be executed.
The -x flag prints commands as they are executed.
The -mod flag's value sets which module download mode
to use: readonly or vendor. See 'go help modules' for more.
To run gofmt with specific options, run gofmt itself.
See also: go fix, go vet.
@ -68,7 +72,7 @@ func runFmt(cmd *base.Command, args []string) {
continue
}
if pkg.Error != nil {
if strings.HasPrefix(pkg.Error.Err, "build constraints exclude all Go files") {
if strings.HasPrefix(pkg.Error.Err.Error(), "build constraints exclude all Go files") {
// Skip this error, as we will format
// all files regardless.
} else {

2
libgo/go/cmd/go/internal/generate/generate.go
View file

@ -149,7 +149,7 @@ var (
)
func init() {
work.AddBuildFlags(CmdGenerate)
work.AddBuildFlags(CmdGenerate, work.DefaultBuildFlags)
CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "")
}

16
libgo/go/cmd/go/internal/get/discovery.go
View file

@ -11,15 +11,15 @@ import (
"strings"
)
// charsetReader returns a reader for the given charset. Currently
// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful
// charsetReader returns a reader that converts from the given charset to UTF-8.
// Currently it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful
// error which is printed by go get, so the user can find why the package
// wasn't downloaded if the encoding is not supported. Note that, in
// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters
// greater than 0x7f are not rejected).
func charsetReader(charset string, input io.Reader) (io.Reader, error) {
switch strings.ToLower(charset) {
case "ascii":
case "utf-8", "ascii":
return input, nil
default:
return nil, fmt.Errorf("can't decode XML document using charset %q", charset)
@ -28,16 +28,16 @@ func charsetReader(charset string, input io.Reader) (io.Reader, error) {
// parseMetaGoImports returns meta imports from the HTML in r.
// Parsing ends at the end of the <head> section or the beginning of the <body>.
func parseMetaGoImports(r io.Reader, mod ModuleMode) (imports []metaImport, err error) {
func parseMetaGoImports(r io.Reader, mod ModuleMode) ([]metaImport, error) {
d := xml.NewDecoder(r)
d.CharsetReader = charsetReader
d.Strict = false
var t xml.Token
var imports []metaImport
for {
t, err = d.RawToken()
t, err := d.RawToken()
if err != nil {
if err == io.EOF || len(imports) > 0 {
err = nil
if err != io.EOF && len(imports) == 0 {
return nil, err
}
break
}

6
libgo/go/cmd/go/internal/get/get.go
View file

@ -108,7 +108,7 @@ var (
)
func init() {
work.AddBuildFlags(CmdGet)
work.AddBuildFlags(CmdGet, work.OmitModFlag|work.OmitModCommonFlags)
CmdGet.Run = runGet // break init loop
CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "")
}
@ -274,7 +274,7 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int)
stk.Push(arg)
err := downloadPackage(p)
if err != nil {
base.Errorf("%s", &load.PackageError{ImportStack: stk.Copy(), Err: err.Error()})
base.Errorf("%s", &load.PackageError{ImportStack: stk.Copy(), Err: err})
stk.Pop()
return
}
@ -355,7 +355,7 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int)
stk.Push(path)
err := &load.PackageError{
ImportStack: stk.Copy(),
Err: "must be imported as " + path[j+len("vendor/"):],
Err: load.ImportErrorf(path, "%s must be imported as %s", path, path[j+len("vendor/"):]),
}
stk.Pop()
base.Errorf("%s", err)

5
libgo/go/cmd/go/internal/get/path.go
View file

@ -11,7 +11,7 @@ import (
"unicode/utf8"
)
// The following functions are copied verbatim from cmd/go/internal/module/module.go,
// The following functions are copied verbatim from golang.org/x/mod/module/module.go,
// with a change to additionally reject Windows short-names,
// and one to accept arbitrary letters (golang.org/issue/29101).
//
@ -44,9 +44,6 @@ func checkPath(path string, fileName bool) error {
if path[0] == '-' {
return fmt.Errorf("leading dash")
}
if strings.Contains(path, "..") {
return fmt.Errorf("double dot")
}
if strings.Contains(path, "//") {
return fmt.Errorf("double slash")
}

78
libgo/go/cmd/go/internal/get/vcs.go
View file

@ -21,6 +21,7 @@ import (
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/load"
"cmd/go/internal/web"
)
@ -531,12 +532,12 @@ func (v *vcsCmd) tagSync(dir, tag string) error {
// A vcsPath describes how to convert an import path into a
// version control system and repository name.
type vcsPath struct {
prefix string // prefix this description applies to
regexp *lazyregexp.Regexp // compiled pattern for import path
repo string // repository to use (expand with match of re)
vcs string // version control system to use (expand with match of re)
check func(match map[string]string) error // additional checks
ping bool // ping for scheme to use to download repo
prefix string // prefix this description applies to
regexp *lazyregexp.Regexp // compiled pattern for import path
repo string // repository to use (expand with match of re)
vcs string // version control system to use (expand with match of re)
check func(match map[string]string) error // additional checks
schemelessRepo bool // if true, the repo pattern lacks a scheme
}
// vcsFromDir inspects dir and its parents to determine the
@ -657,15 +658,15 @@ const (
// RepoRootForImportPath analyzes importPath to determine the
// version control system, and code repository to use.
func RepoRootForImportPath(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths)
rr, err := repoRootFromVCSPaths(importPath, security, vcsPaths)
if err == errUnknownSite {
rr, err = repoRootForImportDynamic(importPath, mod, security)
if err != nil {
err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err)
err = load.ImportErrorf(importPath, "unrecognized import path %q: %v", importPath, err)
}
}
if err != nil {
rr1, err1 := repoRootFromVCSPaths(importPath, "", security, vcsPathsAfterDynamic)
rr1, err1 := repoRootFromVCSPaths(importPath, security, vcsPathsAfterDynamic)
if err1 == nil {
rr = rr1
err = nil
@ -676,7 +677,7 @@ func RepoRootForImportPath(importPath string, mod ModuleMode, security web.Secur
if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") {
// Do not allow wildcards in the repo root.
rr = nil
err = fmt.Errorf("cannot expand ... in %q", importPath)
err = load.ImportErrorf(importPath, "cannot expand ... in %q", importPath)
}
return rr, err
}
@ -685,8 +686,7 @@ var errUnknownSite = errors.New("dynamic lookup required to find mapping")
// repoRootFromVCSPaths attempts to map importPath to a repoRoot
// using the mappings defined in vcsPaths.
// If scheme is non-empty, that scheme is forced.
func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) {
func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) {
// A common error is to use https://packagepath because that's what
// hg and git require. Diagnose this helpfully.
if prefix := httpPrefix(importPath); prefix != "" {
@ -701,7 +701,7 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode,
m := srv.regexp.FindStringSubmatch(importPath)
if m == nil {
if srv.prefix != "" {
return nil, fmt.Errorf("invalid %s import path %q", srv.prefix, importPath)
return nil, load.ImportErrorf(importPath, "invalid %s import path %q", srv.prefix, importPath)
}
continue
}
@ -731,26 +731,28 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode,
if vcs == nil {
return nil, fmt.Errorf("unknown version control system %q", match["vcs"])
}
if srv.ping {
if scheme != "" {
match["repo"] = scheme + "://" + match["repo"]
} else {
for _, scheme := range vcs.scheme {
if security == web.SecureOnly && !vcs.isSecureScheme(scheme) {
var repoURL string
if !srv.schemelessRepo {
repoURL = match["repo"]
} else {
scheme := vcs.scheme[0] // default to first scheme
repo := match["repo"]
if vcs.pingCmd != "" {
// If we know how to test schemes, scan to find one.
for _, s := range vcs.scheme {
if security == web.SecureOnly && !vcs.isSecureScheme(s) {
continue
}
if vcs.pingCmd != "" && vcs.ping(scheme, match["repo"]) == nil {
match["repo"] = scheme + "://" + match["repo"]
goto Found
if vcs.ping(s, repo) == nil {
scheme = s
break
}
}
// No scheme found. Fall back to the first one.
match["repo"] = vcs.scheme[0] + "://" + match["repo"]
Found:
}
repoURL = scheme + "://" + repo
}
rr := &RepoRoot{
Repo: match["repo"],
Repo: repoURL,
Root: match["root"],
VCS: vcs.cmd,
vcs: vcs,
@ -799,6 +801,13 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se
body := resp.Body
defer body.Close()
imports, err := parseMetaGoImports(body, mod)
if len(imports) == 0 {
if respErr := resp.Err(); respErr != nil {
// If the server's status was not OK, prefer to report that instead of
// an XML parse error.
return nil, respErr
}
}
if err != nil {
return nil, fmt.Errorf("parsing %s: %v", importPath, err)
}
@ -904,16 +913,23 @@ func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.Secu
}
resp, err := web.Get(security, url)
if err != nil {
return setCache(fetchResult{url: url, err: fmt.Errorf("fetch %s: %v", resp.URL, err)})
return setCache(fetchResult{url: url, err: fmt.Errorf("fetching %s: %v", importPrefix, err)})
}
body := resp.Body
defer body.Close()
imports, err := parseMetaGoImports(body, mod)
if len(imports) == 0 {
if respErr := resp.Err(); respErr != nil {
// If the server's status was not OK, prefer to report that instead of
// an XML parse error.
return setCache(fetchResult{url: url, err: respErr})
}
}
if err != nil {
return setCache(fetchResult{url: url, err: fmt.Errorf("parsing %s: %v", resp.URL, err)})
}
if len(imports) == 0 {
err = fmt.Errorf("fetch %s: no go-import meta tag", url)
err = fmt.Errorf("fetching %s: no go-import meta tag found in %s", importPrefix, resp.URL)
}
return setCache(fetchResult{url: url, imports: imports, err: err})
})
@ -962,7 +978,7 @@ func (m ImportMismatchError) Error() string {
// matchGoImport returns the metaImport from imports matching importPath.
// An error is returned if there are multiple matches.
// errNoMatch is returned if none match.
// An ImportMismatchError is returned if none match.
func matchGoImport(imports []metaImport, importPath string) (metaImport, error) {
match := -1
@ -1061,8 +1077,8 @@ var vcsPaths = []*vcsPath{
// General syntax for any server.
// Must be last.
{
regexp: lazyregexp.New(`(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`),
ping: true,
regexp: lazyregexp.New(`(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`),
schemelessRepo: true,
},
}

16
libgo/go/cmd/go/internal/help/helpdoc.go
View file

@ -21,8 +21,8 @@ http://swig.org/. When running go build, any file with a .swig
extension will be passed to SWIG. Any file with a .swigcxx extension
will be passed to SWIG with the -c++ option.
When either cgo or SWIG is used, go build will pass any .c, .m, .s,
or .S files to the C compiler, and any .cc, .cpp, .cxx files to the C++
When either cgo or SWIG is used, go build will pass any .c, .m, .s, .S
or .sx files to the C compiler, and any .cc, .cpp, .cxx files to the C++
compiler. The CC or CXX environment variables may be set to determine
the C or C++ compiler, respectively, to use.
`,
@ -506,6 +506,10 @@ General-purpose environment variables:
Because the entries are space-separated, flag values must
not contain spaces. Flags listed on the command line
are applied after this list and therefore override it.
GOINSECURE
Comma-separated list of glob patterns (in the syntax of Go's path.Match)
of module path prefixes that should always be fetched in an insecure
manner. Only applies to dependencies that are being fetched directly.
GOOS
The operating system for which to compile code.
Examples are linux, darwin, windows, netbsd.
@ -617,8 +621,10 @@ Additional information available from 'go env' but not read from the environment
GOHOSTOS
The operating system (GOOS) of the Go toolchain binaries.
GOMOD
The absolute path to the go.mod of the main module,
or the empty string if not using modules.
The absolute path to the go.mod of the main module.
If module-aware mode is enabled, but there is no go.mod, GOMOD will be
os.DevNull ("/dev/null" on Unix-like systems, "NUL" on Windows).
If module-aware mode is disabled, GOMOD will be the empty string.
GOTOOLDIR
The directory where the go tools (compile, cover, doc, etc...) are installed.
`,
@ -645,7 +651,7 @@ the extension of the file name. These extensions are:
.m
Objective-C source files. Only useful with cgo, and always
compiled with the OS-native compiler.
.s, .S
.s, .S, .sx
Assembler source files.
If the package uses cgo or SWIG, these will be assembled with the
OS-native assembler (typically gcc (sic)); otherwise they

4
libgo/go/cmd/go/internal/imports/build.go
View file

@ -210,7 +210,7 @@ var KnownOS = map[string]bool{
"illumos": true,
"js": true,
"linux": true,
"nacl": true,
"nacl": true, // legacy; don't remove
"netbsd": true,
"openbsd": true,
"plan9": true,
@ -222,7 +222,7 @@ var KnownOS = map[string]bool{
var KnownArch = map[string]bool{
"386": true,
"amd64": true,
"amd64p32": true,
"amd64p32": true, // legacy; don't remove
"arm": true,
"armbe": true,
"arm64": true,

18
libgo/go/cmd/go/internal/list/list.go
View file

@ -287,7 +287,7 @@ For more about modules, see 'go help modules'.
func init() {
CmdList.Run = runList // break init cycle
work.AddBuildFlags(CmdList)
work.AddBuildFlags(CmdList, work.DefaultBuildFlags)
}
var (
@ -384,6 +384,22 @@ func runList(cmd *base.Command, args []string) {
if modload.Init(); !modload.Enabled() {
base.Fatalf("go list -m: not using modules")
}
modload.InitMod() // Parses go.mod and sets cfg.BuildMod.
if cfg.BuildMod == "vendor" {
for _, arg := range args {
// In vendor mode, the module graph is incomplete: it contains only the
// explicit module dependencies and the modules that supply packages in
// the import graph. Reject queries that imply more information than that.
if arg == "all" {
base.Fatalf("go list -m: can't compute 'all' using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)")
}
if strings.Contains(arg, "...") {
base.Fatalf("go list -m: can't match module patterns using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)")
}
}
}
modload.LoadBuildList()
mods := modload.ListModules(args, *listU, *listVersions)

251
libgo/go/cmd/go/internal/load/pkg.go
View file

@ -7,9 +7,11 @@ package load
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"go/build"
"go/scanner"
"go/token"
"io/ioutil"
"os"
@ -40,7 +42,7 @@ var (
ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct
ModImportPaths func(args []string) []*search.Match // expand import paths
ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary
ModInfoProg func(info string) []byte // wrap module info in .go code for binary
ModInfoProg func(info string, isgccgo bool) []byte // wrap module info in .go code for binary
ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files
ModDirImportPath func(string) string // return effective import path for directory
)
@ -304,9 +306,9 @@ func (p *Package) copyBuild(pp *build.Package) {
type PackageError struct {
ImportStack []string // shortest path from package named on command line to this one
Pos string // position of error
Err string // the error itself
IsImportCycle bool `json:"-"` // the error is an import cycle
Hard bool `json:"-"` // whether the error is soft or hard; soft errors are ignored in some places
Err error // the error itself
IsImportCycle bool // the error is an import cycle
Hard bool // whether the error is soft or hard; soft errors are ignored in some places
}
func (p *PackageError) Error() string {
@ -317,12 +319,77 @@ func (p *PackageError) Error() string {
if p.Pos != "" {
// Omit import stack. The full path to the file where the error
// is the most important thing.
return p.Pos + ": " + p.Err
return p.Pos + ": " + p.Err.Error()
}
if len(p.ImportStack) == 0 {
return p.Err
// If the error is an ImportPathError, and the last path on the stack appears
// in the error message, omit that path from the stack to avoid repetition.
// If an ImportPathError wraps another ImportPathError that matches the
// last path on the stack, we don't omit the path. An error like
// "package A imports B: error loading C caused by B" would not be clearer
// if "imports B" were omitted.
stack := p.ImportStack
var ierr ImportPathError
if len(stack) > 0 && errors.As(p.Err, &ierr) && ierr.ImportPath() == stack[len(stack)-1] {
stack = stack[:len(stack)-1]
}
return "package " + strings.Join(p.ImportStack, "\n\timports ") + ": " + p.Err
if len(stack) == 0 {
return p.Err.Error()
}
return "package " + strings.Join(stack, "\n\timports ") + ": " + p.Err.Error()
}
// PackageError implements MarshalJSON so that Err is marshaled as a string
// and non-essential fields are omitted.
func (p *PackageError) MarshalJSON() ([]byte, error) {
perr := struct {
ImportStack []string
Pos string
Err string
}{p.ImportStack, p.Pos, p.Err.Error()}
return json.Marshal(perr)
}
// ImportPathError is a type of error that prevents a package from being loaded
// for a given import path. When such a package is loaded, a *Package is
// returned with Err wrapping an ImportPathError: the error is attached to
// the imported package, not the importing package.
//
// The string returned by ImportPath must appear in the string returned by
// Error. Errors that wrap ImportPathError (such as PackageError) may omit
// the import path.
type ImportPathError interface {
error
ImportPath() string
}
type importError struct {
importPath string
err error // created with fmt.Errorf
}
var _ ImportPathError = (*importError)(nil)
func ImportErrorf(path, format string, args ...interface{}) ImportPathError {
err := &importError{importPath: path, err: fmt.Errorf(format, args...)}
if errStr := err.Error(); !strings.Contains(errStr, path) {
panic(fmt.Sprintf("path %q not in error %q", path, errStr))
}
return err
}
func (e *importError) Error() string {
return e.err.Error()
}
func (e *importError) Unwrap() error {
// Don't return e.err directly, since we're only wrapping an error if %w
// was passed to ImportErrorf.
return errors.Unwrap(e.err)
}
func (e *importError) ImportPath() string {
return e.importPath
}
// An ImportStack is a stack of import paths, possibly with the suffix " (test)" appended.
@ -489,7 +556,7 @@ func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportS
ImportPath: path,
Error: &PackageError{
ImportStack: stk.Copy(),
Err: err.Error(),
Err: err,
},
},
}
@ -516,7 +583,7 @@ func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportS
if !cfg.ModulesEnabled && path != cleanImport(path) {
p.Error = &PackageError{
ImportStack: stk.Copy(),
Err: fmt.Sprintf("non-canonical import path: %q should be %q", path, pathpkg.Clean(path)),
Err: fmt.Errorf("non-canonical import path: %q should be %q", path, pathpkg.Clean(path)),
}
p.Incomplete = true
}
@ -527,7 +594,7 @@ func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportS
return setErrorPos(perr, importPos)
}
if mode&ResolveImport != 0 {
if perr := disallowVendor(srcDir, parent, parentPath, path, p, stk); perr != p {
if perr := disallowVendor(srcDir, path, p, stk); perr != p {
return setErrorPos(perr, importPos)
}
}
@ -536,20 +603,22 @@ func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportS
perr := *p
perr.Error = &PackageError{
ImportStack: stk.Copy(),
Err: fmt.Sprintf("import %q is a program, not an importable package", path),
Err: ImportErrorf(path, "import %q is a program, not an importable package", path),
}
return setErrorPos(&perr, importPos)
}
if p.Internal.Local && parent != nil && !parent.Internal.Local {
perr := *p
errMsg := fmt.Sprintf("local import %q in non-local package", path)
var err error
if path == "." {
errMsg = "cannot import current directory"
err = ImportErrorf(path, "%s: cannot import current directory", path)
} else {
err = ImportErrorf(path, "local import %q in non-local package", path)
}
perr.Error = &PackageError{
ImportStack: stk.Copy(),
Err: errMsg,
Err: err,
}
return setErrorPos(&perr, importPos)
}
@ -602,6 +671,11 @@ func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd
// we create from the full directory to the package.
// Otherwise it is the usual import path.
// For vendored imports, it is the expanded form.
//
// Note that when modules are enabled, local import paths are normally
// canonicalized by modload.ImportPaths before now. However, if there's an
// error resolving a local path, it will be returned untransformed
// so that 'go list -e' reports something useful.
importKey := importSpec{
path: path,
parentPath: parentPath,
@ -1125,7 +1199,7 @@ func reusePackage(p *Package, stk *ImportStack) *Package {
if p.Error == nil {
p.Error = &PackageError{
ImportStack: stk.Copy(),
Err: "import cycle not allowed",
Err: errors.New("import cycle not allowed"),
IsImportCycle: true,
}
}
@ -1234,7 +1308,7 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p *
perr := *p
perr.Error = &PackageError{
ImportStack: stk.Copy(),
Err: "use of internal package " + p.ImportPath + " not allowed",
Err: ImportErrorf(p.ImportPath, "use of internal package "+p.ImportPath+" not allowed"),
}
perr.Incomplete = true
return &perr
@ -1259,11 +1333,10 @@ func findInternal(path string) (index int, ok bool) {
return 0, false
}
// disallowVendor checks that srcDir (containing package importerPath, if non-empty)
// is allowed to import p as path.
// disallowVendor checks that srcDir is allowed to import p as path.
// If the import is allowed, disallowVendor returns the original package p.
// If not, it returns a new package containing just an appropriate error.
func disallowVendor(srcDir string, importer *Package, importerPath, path string, p *Package, stk *ImportStack) *Package {
func disallowVendor(srcDir string, path string, p *Package, stk *ImportStack) *Package {
// The stack includes p.ImportPath.
// If that's the only thing on the stack, we started
// with a name given on the command line, not an
@ -1281,7 +1354,7 @@ func disallowVendor(srcDir string, importer *Package, importerPath, path string,
perr := *p
perr.Error = &PackageError{
ImportStack: stk.Copy(),
Err: "must be imported as " + path[i+len("vendor/"):],
Err: ImportErrorf(path, "%s must be imported as %s", path, path[i+len("vendor/"):]),
}
perr.Incomplete = true
return &perr
@ -1335,7 +1408,7 @@ func disallowVendorVisibility(srcDir string, p *Package, stk *ImportStack) *Pack
perr := *p
perr.Error = &PackageError{
ImportStack: stk.Copy(),
Err: "use of vendored package not allowed",
Err: errors.New("use of vendored package not allowed"),
}
perr.Incomplete = true
return &perr
@ -1397,26 +1470,51 @@ var cgoSyscallExclude = map[string]bool{
var foldPath = make(map[string]string)
// DefaultExecName returns the default executable name
// for a package with the import path importPath.
// exeFromImportPath returns an executable name
// for a package using the import path.
//
// The default executable name is the last element of the import path.
// The executable name is the last element of the import path.
// In module-aware mode, an additional rule is used on import paths
// consisting of two or more path elements. If the last element is
// a vN path element specifying the major version, then the
// second last element of the import path is used instead.
func DefaultExecName(importPath string) string {
_, elem := pathpkg.Split(importPath)
func (p *Package) exeFromImportPath() string {
_, elem := pathpkg.Split(p.ImportPath)
if cfg.ModulesEnabled {
// If this is example.com/mycmd/v2, it's more useful to
// install it as mycmd than as v2. See golang.org/issue/24667.
if elem != importPath && isVersionElement(elem) {
_, elem = pathpkg.Split(pathpkg.Dir(importPath))
if elem != p.ImportPath && isVersionElement(elem) {
_, elem = pathpkg.Split(pathpkg.Dir(p.ImportPath))
}
}
return elem
}
// exeFromFiles returns an executable name for a package
// using the first element in GoFiles or CgoFiles collections without the prefix.
//
// Returns empty string in case of empty collection.
func (p *Package) exeFromFiles() string {
var src string
if len(p.GoFiles) > 0 {
src = p.GoFiles[0]
} else if len(p.CgoFiles) > 0 {
src = p.CgoFiles[0]
} else {
return ""
}
_, elem := filepath.Split(src)
return elem[:len(elem)-len(".go")]
}
// DefaultExecName returns the default executable name for a package
func (p *Package) DefaultExecName() string {
if p.Internal.CmdlineFiles {
return p.exeFromFiles()
}
return p.exeFromImportPath()
}
// load populates p using information from bp, err, which should
// be the result of calling build.Context.Import.
func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
@ -1428,17 +1526,30 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
p.Internal.LocalPrefix = dirToImportPath(p.Dir)
}
// setError sets p.Error if it hasn't already been set. We may proceed
// after encountering some errors so that 'go list -e' has more complete
// output. If there's more than one error, we should report the first.
setError := func(err error) {
if p.Error == nil {
p.Error = &PackageError{
ImportStack: stk.Copy(),
Err: err,
}
}
}
if err != nil {
if _, ok := err.(*build.NoGoError); ok {
err = &NoGoError{Package: p}
}
p.Incomplete = true
err = base.ExpandScanner(err)
p.Error = &PackageError{
ImportStack: stk.Copy(),
Err: err.Error(),
setError(base.ExpandScanner(err))
if _, isScanErr := err.(scanner.ErrorList); !isScanErr {
return
}
return
// Fall through if there was an error parsing a file. 'go list -e' should
// still report imports and other metadata.
}
useBindir := p.Name == "main"
@ -1453,11 +1564,11 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
// Report an error when the old code.google.com/p/go.tools paths are used.
if InstallTargetDir(p) == StalePath {
newPath := strings.Replace(p.ImportPath, "code.google.com/p/go.", "golang.org/x/", 1)
e := fmt.Sprintf("the %v command has moved; use %v instead.", p.ImportPath, newPath)
p.Error = &PackageError{Err: e}
e := ImportErrorf(p.ImportPath, "the %v command has moved; use %v instead.", p.ImportPath, newPath)
setError(e)
return
}
elem := DefaultExecName(p.ImportPath)
elem := p.DefaultExecName()
full := cfg.BuildContext.GOOS + "_" + cfg.BuildContext.GOARCH + "/" + elem
if cfg.BuildContext.GOOS != base.ToolGOOS || cfg.BuildContext.GOARCH != base.ToolGOARCH {
// Install cross-compiled binaries to subdirectories of bin.
@ -1493,7 +1604,10 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
p.Target = ""
} else {
p.Target = p.Internal.Build.PkgObj
if cfg.BuildLinkshared {
if cfg.BuildLinkshared && p.Target != "" {
// TODO(bcmills): The reliance on p.Target implies that -linkshared does
// not work for any package that lacks a Target — such as a non-main
// package in module mode. We should probably fix that.
shlibnamefile := p.Target[:len(p.Target)-2] + ".shlibname"
shlib, err := ioutil.ReadFile(shlibnamefile)
if err != nil && !os.IsNotExist(err) {
@ -1564,10 +1678,7 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
inputs := p.AllFiles()
f1, f2 := str.FoldDup(inputs)
if f1 != "" {
p.Error = &PackageError{
ImportStack: stk.Copy(),
Err: fmt.Sprintf("case-insensitive file name collision: %q and %q", f1, f2),
}
setError(fmt.Errorf("case-insensitive file name collision: %q and %q", f1, f2))
return
}
@ -1580,25 +1691,16 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
// so we shouldn't see any _cgo_ files anyway, but just be safe.
for _, file := range inputs {
if !SafeArg(file) || strings.HasPrefix(file, "_cgo_") {
p.Error = &PackageError{
ImportStack: stk.Copy(),
Err: fmt.Sprintf("invalid input file name %q", file),
}
setError(fmt.Errorf("invalid input file name %q", file))
return
}
}
if name := pathpkg.Base(p.ImportPath); !SafeArg(name) {
p.Error = &PackageError{
ImportStack: stk.Copy(),
Err: fmt.Sprintf("invalid input directory name %q", name),
}
setError(fmt.Errorf("invalid input directory name %q", name))
return
}
if !SafeArg(p.ImportPath) {
p.Error = &PackageError{
ImportStack: stk.Copy(),
Err: fmt.Sprintf("invalid import path %q", p.ImportPath),
}
setError(ImportErrorf(p.ImportPath, "invalid import path %q", p.ImportPath))
return
}
@ -1643,31 +1745,24 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
// code; see issue #16050).
}
setError := func(msg string) {
p.Error = &PackageError{
ImportStack: stk.Copy(),
Err: msg,
}
}
// The gc toolchain only permits C source files with cgo or SWIG.
if len(p.CFiles) > 0 && !p.UsesCgo() && !p.UsesSwig() && cfg.BuildContext.Compiler == "gc" {
setError(fmt.Sprintf("C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CFiles, " ")))
setError(fmt.Errorf("C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CFiles, " ")))
return
}
// C++, Objective-C, and Fortran source files are permitted only with cgo or SWIG,
// regardless of toolchain.
if len(p.CXXFiles) > 0 && !p.UsesCgo() && !p.UsesSwig() {
setError(fmt.Sprintf("C++ source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CXXFiles, " ")))
setError(fmt.Errorf("C++ source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CXXFiles, " ")))
return
}
if len(p.MFiles) > 0 && !p.UsesCgo() && !p.UsesSwig() {
setError(fmt.Sprintf("Objective-C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.MFiles, " ")))
setError(fmt.Errorf("Objective-C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.MFiles, " ")))
return
}
if len(p.FFiles) > 0 && !p.UsesCgo() && !p.UsesSwig() {
setError(fmt.Sprintf("Fortran source files not allowed when not using cgo or SWIG: %s", strings.Join(p.FFiles, " ")))
setError(fmt.Errorf("Fortran source files not allowed when not using cgo or SWIG: %s", strings.Join(p.FFiles, " ")))
return
}
@ -1676,17 +1771,17 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
if other := foldPath[fold]; other == "" {
foldPath[fold] = p.ImportPath
} else if other != p.ImportPath {
setError(fmt.Sprintf("case-insensitive import collision: %q and %q", p.ImportPath, other))
setError(ImportErrorf(p.ImportPath, "case-insensitive import collision: %q and %q", p.ImportPath, other))
return
}
if cfg.ModulesEnabled {
if cfg.ModulesEnabled && p.Error == nil {
mainPath := p.ImportPath
if p.Internal.CmdlineFiles {
mainPath = "command-line-arguments"
}
p.Module = ModPackageModuleInfo(mainPath)
if p.Name == "main" {
if p.Name == "main" && len(p.DepsErrors) == 0 {
p.Internal.BuildInfo = ModPackageBuildInfo(mainPath, p.Deps)
}
}
@ -1956,9 +2051,14 @@ func Packages(args []string) []*Package {
// cannot be loaded at all.
// The packages that fail to load will have p.Error != nil.
func PackagesAndErrors(patterns []string) []*Package {
if len(patterns) > 0 {
for _, p := range patterns {
if strings.HasSuffix(p, ".go") {
for _, p := range patterns {
// Listing is only supported with all patterns referring to either:
// - Files that are part of the same directory.
// - Explicit package paths or patterns.
if strings.HasSuffix(p, ".go") {
// We need to test whether the path is an actual Go file and not a
// package path or pattern ending in '.go' (see golang.org/issue/34653).
if fi, err := os.Stat(p); err == nil && !fi.IsDir() {
return []*Package{GoFilesPackage(patterns)}
}
}
@ -2078,7 +2178,7 @@ func GoFilesPackage(gofiles []string) *Package {
pkg.Internal.CmdlineFiles = true
pkg.Name = f
pkg.Error = &PackageError{
Err: fmt.Sprintf("named files must be .go files: %s", pkg.Name),
Err: fmt.Errorf("named files must be .go files: %s", pkg.Name),
}
return pkg
}
@ -2141,11 +2241,8 @@ func GoFilesPackage(gofiles []string) *Package {
pkg.Match = gofiles
if pkg.Name == "main" {
_, elem := filepath.Split(gofiles[0])
exe := elem[:len(elem)-len(".go")] + cfg.ExeSuffix
if cfg.BuildO == "" {
cfg.BuildO = exe
}
exe := pkg.DefaultExecName() + cfg.ExeSuffix
if cfg.GOBIN != "" {
pkg.Target = filepath.Join(cfg.GOBIN, exe)
} else if cfg.ModulesEnabled {

44
libgo/go/cmd/go/internal/load/pkg_test.go
View file

@ -5,39 +5,49 @@ import (
"testing"
)
func TestDefaultExecName(t *testing.T) {
func TestPkgDefaultExecName(t *testing.T) {
oldModulesEnabled := cfg.ModulesEnabled
defer func() { cfg.ModulesEnabled = oldModulesEnabled }()
for _, tt := range []struct {
in string
files []string
wantMod string
wantGopath string
}{
{"example.com/mycmd", "mycmd", "mycmd"},
{"example.com/mycmd/v0", "v0", "v0"},
{"example.com/mycmd/v1", "v1", "v1"},
{"example.com/mycmd/v2", "mycmd", "v2"}, // Semantic import versioning, use second last element in module mode.
{"example.com/mycmd/v3", "mycmd", "v3"}, // Semantic import versioning, use second last element in module mode.
{"mycmd", "mycmd", "mycmd"},
{"mycmd/v0", "v0", "v0"},
{"mycmd/v1", "v1", "v1"},
{"mycmd/v2", "mycmd", "v2"}, // Semantic import versioning, use second last element in module mode.
{"v0", "v0", "v0"},
{"v1", "v1", "v1"},
{"v2", "v2", "v2"},
{"example.com/mycmd", []string{}, "mycmd", "mycmd"},
{"example.com/mycmd/v0", []string{}, "v0", "v0"},
{"example.com/mycmd/v1", []string{}, "v1", "v1"},
{"example.com/mycmd/v2", []string{}, "mycmd", "v2"}, // Semantic import versioning, use second last element in module mode.
{"example.com/mycmd/v3", []string{}, "mycmd", "v3"}, // Semantic import versioning, use second last element in module mode.
{"mycmd", []string{}, "mycmd", "mycmd"},
{"mycmd/v0", []string{}, "v0", "v0"},
{"mycmd/v1", []string{}, "v1", "v1"},
{"mycmd/v2", []string{}, "mycmd", "v2"}, // Semantic import versioning, use second last element in module mode.
{"v0", []string{}, "v0", "v0"},
{"v1", []string{}, "v1", "v1"},
{"v2", []string{}, "v2", "v2"},
{"command-line-arguments", []string{"output.go", "foo.go"}, "output", "output"},
} {
{
cfg.ModulesEnabled = true
gotMod := DefaultExecName(tt.in)
pkg := new(Package)
pkg.ImportPath = tt.in
pkg.GoFiles = tt.files
pkg.Internal.CmdlineFiles = len(tt.files) > 0
gotMod := pkg.DefaultExecName()
if gotMod != tt.wantMod {
t.Errorf("DefaultExecName(%q) in module mode = %v; want %v", tt.in, gotMod, tt.wantMod)
t.Errorf("pkg.DefaultExecName with ImportPath = %q in module mode = %v; want %v", tt.in, gotMod, tt.wantMod)
}
}
{
cfg.ModulesEnabled = false
gotGopath := DefaultExecName(tt.in)
pkg := new(Package)
pkg.ImportPath = tt.in
pkg.GoFiles = tt.files
pkg.Internal.CmdlineFiles = len(tt.files) > 0
gotGopath := pkg.DefaultExecName()
if gotGopath != tt.wantGopath {
t.Errorf("DefaultExecName(%q) in gopath mode = %v; want %v", tt.in, gotGopath, tt.wantGopath)
t.Errorf("pkg.DefaultExecName with ImportPath = %q in gopath mode = %v; want %v", tt.in, gotGopath, tt.wantGopath)
}
}
}

17
libgo/go/cmd/go/internal/load/test.go
View file

@ -110,7 +110,7 @@ func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest *
// non-test copy of a package.
ptestErr = &PackageError{
ImportStack: testImportStack(stk[0], p1, p.ImportPath),
Err: "import cycle not allowed in test",
Err: errors.New("import cycle not allowed in test"),
IsImportCycle: true,
}
}
@ -271,7 +271,7 @@ func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest *
// afterward that gathers t.Cover information.
t, err := loadTestFuncs(ptest)
if err != nil && pmain.Error == nil {
pmain.Error = &PackageError{Err: err.Error()}
pmain.Error = &PackageError{Err: err}
}
t.Cover = cover
if len(ptest.GoFiles)+len(ptest.CgoFiles) > 0 {
@ -322,7 +322,7 @@ func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest *
data, err := formatTestmain(t)
if err != nil && pmain.Error == nil {
pmain.Error = &PackageError{Err: err.Error()}
pmain.Error = &PackageError{Err: err}
}
if data != nil {
pmain.Internal.TestmainGo = &data
@ -399,10 +399,13 @@ func recompileForTest(pmain, preal, ptest, pxtest *Package) {
}
}
// Don't compile build info from a main package. This can happen
// if -coverpkg patterns include main packages, since those packages
// are imported by pmain. See golang.org/issue/30907.
if p.Internal.BuildInfo != "" && p != pmain {
// Force main packages the test imports to be built as libraries.
// Normal imports of main packages are forbidden by the package loader,
// but this can still happen if -coverpkg patterns include main packages:
// covered packages are imported by pmain. Linking multiple packages
// compiled with '-p main' causes duplicate symbol errors.
// See golang.org/issue/30907, golang.org/issue/34114.
if p.Name == "main" && p != pmain && p != ptest {
split()
}
}

2
libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go
View file

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !js,!nacl,!plan9
// +build !js,!plan9
package filelock_test

65
libgo/go/cmd/go/internal/lockedfile/lockedfile.go
View file

@ -120,3 +120,68 @@ func Write(name string, content io.Reader, perm os.FileMode) (err error) {
}
return err
}
// Transform invokes t with the result of reading the named file, with its lock
// still held.
//
// If t returns a nil error, Transform then writes the returned contents back to
// the file, making a best effort to preserve existing contents on error.
//
// t must not modify the slice passed to it.
func Transform(name string, t func([]byte) ([]byte, error)) (err error) {
f, err := Edit(name)
if err != nil {
return err
}
defer f.Close()
old, err := ioutil.ReadAll(f)
if err != nil {
return err
}
new, err := t(old)
if err != nil {
return err
}
if len(new) > len(old) {
// The overall file size is increasing, so write the tail first: if we're
// about to run out of space on the disk, we would rather detect that
// failure before we have overwritten the original contents.
if _, err := f.WriteAt(new[len(old):], int64(len(old))); err != nil {
// Make a best effort to remove the incomplete tail.
f.Truncate(int64(len(old)))
return err
}
}
// We're about to overwrite the old contents. In case of failure, make a best
// effort to roll back before we close the file.
defer func() {
if err != nil {
if _, err := f.WriteAt(old, 0); err == nil {
f.Truncate(int64(len(old)))
}
}
}()
if len(new) >= len(old) {
if _, err := f.WriteAt(new[:len(old)], 0); err != nil {
return err
}
} else {
if _, err := f.WriteAt(new, 0); err != nil {
return err
}
// The overall file size is decreasing, so shrink the file to its final size
// after writing. We do this after writing (instead of before) so that if
// the write fails, enough filesystem space will likely still be reserved
// to contain the previous contents.
if err := f.Truncate(int64(len(new))); err != nil {
return err
}
}
return nil
}

4
libgo/go/cmd/go/internal/lockedfile/lockedfile_test.go
View file

@ -2,8 +2,8 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// js and nacl do not support inter-process file locking.
// +build !js,!nacl
// js does not support inter-process file locking.
// +build !js
package lockedfile_test

104
libgo/go/cmd/go/internal/lockedfile/transform_test.go Normal file
View file

@ -0,0 +1,104 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// js does not support inter-process file locking.
// +build !js
package lockedfile_test
import (
"bytes"
"encoding/binary"
"math/rand"
"path/filepath"
"testing"
"time"
"cmd/go/internal/lockedfile"
)
func isPowerOf2(x int) bool {
return x > 0 && x&(x-1) == 0
}
func roundDownToPowerOf2(x int) int {
if x <= 0 {
panic("nonpositive x")
}
bit := 1
for x != bit {
x = x &^ bit
bit <<= 1
}
return x
}
func TestTransform(t *testing.T) {
dir, remove := mustTempDir(t)
defer remove()
path := filepath.Join(dir, "blob.bin")
const maxChunkWords = 8 << 10
buf := make([]byte, 2*maxChunkWords*8)
for i := uint64(0); i < 2*maxChunkWords; i++ {
binary.LittleEndian.PutUint64(buf[i*8:], i)
}
if err := lockedfile.Write(path, bytes.NewReader(buf[:8]), 0666); err != nil {
t.Fatal(err)
}
var attempts int64 = 128
if !testing.Short() {
attempts *= 16
}
const parallel = 32
var sem = make(chan bool, parallel)
for n := attempts; n > 0; n-- {
sem <- true
go func() {
defer func() { <-sem }()
time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond)
chunkWords := roundDownToPowerOf2(rand.Intn(maxChunkWords) + 1)
offset := rand.Intn(chunkWords)
err := lockedfile.Transform(path, func(data []byte) (chunk []byte, err error) {
chunk = buf[offset*8 : (offset+chunkWords)*8]
if len(data)&^7 != len(data) {
t.Errorf("read %d bytes, but each write is an integer multiple of 8 bytes", len(data))
return chunk, nil
}
words := len(data) / 8
if !isPowerOf2(words) {
t.Errorf("read %d 8-byte words, but each write is a power-of-2 number of words", words)
return chunk, nil
}
u := binary.LittleEndian.Uint64(data)
for i := 1; i < words; i++ {
next := binary.LittleEndian.Uint64(data[i*8:])
if next != u+1 {
t.Errorf("wrote sequential integers, but read integer out of sequence at offset %d", i)
return chunk, nil
}
u = next
}
return chunk, nil
})
if err != nil {
t.Errorf("unexpected error from Transform: %v", err)
}
}()
}
for n := parallel; n > 0; n-- {
sem <- true
}
}

58
libgo/go/cmd/go/internal/modcmd/download.go
View file

@ -5,19 +5,21 @@
package modcmd
import (
"cmd/go/internal/cfg"
"encoding/json"
"os"
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/modfetch"
"cmd/go/internal/modload"
"cmd/go/internal/module"
"cmd/go/internal/par"
"cmd/go/internal/work"
"golang.org/x/mod/module"
)
var cmdDownload = &base.Command{
UsageLine: "go mod download [-json] [modules]",
UsageLine: "go mod download [-x] [-json] [modules]",
Short: "download modules to local cache",
Long: `
Download downloads the named modules, which can be module patterns selecting
@ -43,9 +45,10 @@ corresponding to this Go struct:
Dir string // absolute path to cached source root directory
Sum string // checksum for path, version (as in go.sum)
GoModSum string // checksum for go.mod (as in go.sum)
Latest bool // would @latest resolve to this version?
}
The -x flag causes download to print the commands download executes.
See 'go help modules' for more about module queries.
`,
}
@ -54,6 +57,10 @@ var downloadJSON = cmdDownload.Flag.Bool("json", false, "")
func init() {
cmdDownload.Run = runDownload // break init cycle
// TODO(jayconrod): https://golang.org/issue/35849 Apply -x to other 'go mod' commands.
cmdDownload.Flag.BoolVar(&cfg.BuildX, "x", false, "")
work.AddModCommonFlags(cmdDownload)
}
type moduleJSON struct {
@ -66,7 +73,6 @@ type moduleJSON struct {
Dir string `json:",omitempty"`
Sum string `json:",omitempty"`
GoModSum string `json:",omitempty"`
Latest bool `json:",omitempty"`
}
func runDownload(cmd *base.Command, args []string) {
@ -79,6 +85,17 @@ func runDownload(cmd *base.Command, args []string) {
}
if len(args) == 0 {
args = []string{"all"}
} else if modload.HasModRoot() {
modload.InitMod() // to fill Target
targetAtLatest := modload.Target.Path + "@latest"
targetAtUpgrade := modload.Target.Path + "@upgrade"
targetAtPatch := modload.Target.Path + "@patch"
for _, arg := range args {
switch arg {
case modload.Target.Path, targetAtLatest, targetAtUpgrade, targetAtPatch:
os.Stderr.WriteString("go mod download: skipping argument " + arg + " that resolves to the main module\n")
}
}
}
var mods []*moduleJSON
@ -90,7 +107,8 @@ func runDownload(cmd *base.Command, args []string) {
info = info.Replace
}
if info.Version == "" && info.Error == nil {
// main module
// main module or module replaced with file path.
// Nothing to download.
continue
}
m := &moduleJSON{
@ -105,31 +123,6 @@ func runDownload(cmd *base.Command, args []string) {
work.Add(m)
}
latest := map[string]string{} // path → version
if *downloadJSON {
// We need to populate the Latest field, but if the main module depends on a
// version newer than latest — or if the version requested on the command
// line is itself newer than latest — that's not trivial to determine from
// the info returned by ListModules. Instead, we issue a separate
// ListModules request for "latest", which should be inexpensive relative to
// downloading the modules.
var latestArgs []string
for _, m := range mods {
if m.Error != "" {
continue
}
latestArgs = append(latestArgs, m.Path+"@latest")
}
if len(latestArgs) > 0 {
for _, info := range modload.ListModules(latestArgs, listU, listVersions) {
if info.Version != "" {
latest[info.Path] = info.Version
}
}
}
}
work.Do(10, func(item interface{}) {
m := item.(*moduleJSON)
var err error
@ -160,9 +153,6 @@ func runDownload(cmd *base.Command, args []string) {
m.Error = err.Error()
return
}
if latest[m.Path] == m.Version {
m.Latest = true
}
})
if *downloadJSON {

58
libgo/go/cmd/go/internal/modcmd/edit.go
View file

@ -9,17 +9,19 @@ package modcmd
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"cmd/go/internal/base"
"cmd/go/internal/lockedfile"
"cmd/go/internal/modfetch"
"cmd/go/internal/modfile"
"cmd/go/internal/modload"
"cmd/go/internal/module"
"cmd/go/internal/work"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
var cmdEdit = &base.Command{
@ -53,12 +55,17 @@ The -exclude=path@version and -dropexclude=path@version flags
add and drop an exclusion for the given module path and version.
Note that -exclude=path@version is a no-op if that exclusion already exists.
The -replace=old[@v]=new[@v] and -dropreplace=old[@v] flags
add and drop a replacement of the given module path and version pair.
If the @v in old@v is omitted, the replacement applies to all versions
with the old module path. If the @v in new@v is omitted, the new path
should be a local module root directory, not a module path.
Note that -replace overrides any existing replacements for old[@v].
The -replace=old[@v]=new[@v] flag adds a replacement of the given
module path and version pair. If the @v in old@v is omitted, a
replacement without a version on the left side is added, which applies
to all versions of the old module path. If the @v in new@v is omitted,
the new path should be a local module root directory, not a module
path. Note that -replace overrides any redundant replacements for old[@v],
so omitting @v will drop existing replacements for specific versions.
The -dropreplace=old[@v] flag drops a replacement of the given
module path and version pair. If the @v is omitted, a replacement without
a version on the left side is dropped.
The -require, -droprequire, -exclude, -dropexclude, -replace,
and -dropreplace editing flags may be repeated, and the changes
@ -130,6 +137,7 @@ func init() {
cmdEdit.Flag.Var(flagFunc(flagReplace), "replace", "")
cmdEdit.Flag.Var(flagFunc(flagDropExclude), "dropexclude", "")
work.AddModCommonFlags(cmdEdit)
base.AddBuildFlagsNX(&cmdEdit.Flag)
}
@ -157,11 +165,11 @@ func runEdit(cmd *base.Command, args []string) {
if len(args) == 1 {
gomod = args[0]
} else {
gomod = filepath.Join(modload.ModRoot(), "go.mod")
gomod = modload.ModFilePath()
}
if *editModule != "" {
if err := module.CheckPath(*editModule); err != nil {
if err := module.CheckImportPath(*editModule); err != nil {
base.Fatalf("go mod: invalid -module: %v", err)
}
}
@ -172,7 +180,7 @@ func runEdit(cmd *base.Command, args []string) {
}
}
data, err := ioutil.ReadFile(gomod)
data, err := lockedfile.Read(gomod)
if err != nil {
base.Fatalf("go: %v", err)
}
@ -215,13 +223,19 @@ func runEdit(cmd *base.Command, args []string) {
return
}
unlock := modfetch.SideLock()
defer unlock()
lockedData, err := ioutil.ReadFile(gomod)
if err == nil && !bytes.Equal(lockedData, data) {
base.Fatalf("go: go.mod changed during editing; not overwriting")
// Make a best-effort attempt to acquire the side lock, only to exclude
// previous versions of the 'go' command from making simultaneous edits.
if unlock, err := modfetch.SideLock(); err == nil {
defer unlock()
}
if err := ioutil.WriteFile(gomod, out, 0666); err != nil {
err = lockedfile.Transform(gomod, func(lockedData []byte) ([]byte, error) {
if !bytes.Equal(lockedData, data) {
return nil, errors.New("go.mod changed during editing; not overwriting")
}
return out, nil
})
if err != nil {
base.Fatalf("go: %v", err)
}
}
@ -233,7 +247,7 @@ func parsePathVersion(flag, arg string) (path, version string) {
base.Fatalf("go mod: -%s=%s: need path@version", flag, arg)
}
path, version = strings.TrimSpace(arg[:i]), strings.TrimSpace(arg[i+1:])
if err := module.CheckPath(path); err != nil {
if err := module.CheckImportPath(path); err != nil {
base.Fatalf("go mod: -%s=%s: invalid path: %v", flag, arg, err)
}
@ -255,7 +269,7 @@ func parsePath(flag, arg string) (path string) {
base.Fatalf("go mod: -%s=%s: need just path, not path@version", flag, arg)
}
path = arg
if err := module.CheckPath(path); err != nil {
if err := module.CheckImportPath(path); err != nil {
base.Fatalf("go mod: -%s=%s: invalid path: %v", flag, arg, err)
}
return path
@ -269,7 +283,7 @@ func parsePathVersionOptional(adj, arg string, allowDirPath bool) (path, version
} else {
path, version = strings.TrimSpace(arg[:i]), strings.TrimSpace(arg[i+1:])
}
if err := module.CheckPath(path); err != nil {
if err := module.CheckImportPath(path); err != nil {
if !allowDirPath || !modfile.IsDirectoryPath(path) {
return path, version, fmt.Errorf("invalid %s path: %v", adj, err)
}

10
libgo/go/cmd/go/internal/modcmd/graph.go
View file

@ -8,14 +8,16 @@ package modcmd
import (
"bufio"
"cmd/go/internal/cfg"
"os"
"sort"
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/modload"
"cmd/go/internal/module"
"cmd/go/internal/par"
"cmd/go/internal/work"
"golang.org/x/mod/module"
)
var cmdGraph = &base.Command{
@ -30,6 +32,10 @@ path@version, except for the main module, which has no @version suffix.
Run: runGraph,
}
func init() {
work.AddModCommonFlags(cmdGraph)
}
func runGraph(cmd *base.Command, args []string) {
if len(args) > 0 {
base.Fatalf("go mod graph: graph takes no arguments")

8
libgo/go/cmd/go/internal/modcmd/init.go
View file

@ -9,6 +9,7 @@ package modcmd
import (
"cmd/go/internal/base"
"cmd/go/internal/modload"
"cmd/go/internal/work"
"os"
"strings"
)
@ -27,6 +28,10 @@ To override this guess, supply the module path as an argument.
Run: runInit,
}
func init() {
work.AddModCommonFlags(cmdInit)
}
func runInit(cmd *base.Command, args []string) {
modload.CmdModInit = true
if len(args) > 1 {
@ -38,7 +43,8 @@ func runInit(cmd *base.Command, args []string) {
if os.Getenv("GO111MODULE") == "off" {
base.Fatalf("go mod init: modules disabled by GO111MODULE=off; see 'go help modules'")
}
if _, err := os.Stat("go.mod"); err == nil {
modFilePath := modload.ModFilePath()
if _, err := os.Stat(modFilePath); err == nil {
base.Fatalf("go mod init: go.mod already exists")
}
if strings.Contains(modload.CmdModModule, "@") {

9
libgo/go/cmd/go/internal/modcmd/mod.go
View file

@ -5,7 +5,10 @@
// Package modcmd implements the ``go mod'' command.
package modcmd
import "cmd/go/internal/base"
import (
"cmd/go/internal/base"
"cmd/go/internal/cfg"
)
var CmdMod = &base.Command{
UsageLine: "go mod",
@ -29,3 +32,7 @@ See 'go help modules' for an overview of module functionality.
cmdWhy,
},
}
func addModFlags(cmd *base.Command) {
cmd.Flag.StringVar(&cfg.ModFile, "modfile", "", "")
}

32
libgo/go/cmd/go/internal/modcmd/tidy.go
View file

@ -7,14 +7,13 @@
package modcmd
import (
"fmt"
"os"
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/modfetch"
"cmd/go/internal/modload"
"cmd/go/internal/module"
"cmd/go/internal/work"
"golang.org/x/mod/module"
)
var cmdTidy = &base.Command{
@ -35,6 +34,7 @@ to standard error.
func init() {
cmdTidy.Run = runTidy // break init cycle
cmdTidy.Flag.BoolVar(&cfg.BuildV, "v", false, "")
work.AddModCommonFlags(cmdTidy)
}
func runTidy(cmd *base.Command, args []string) {
@ -42,28 +42,8 @@ func runTidy(cmd *base.Command, args []string) {
base.Fatalf("go mod tidy: no arguments allowed")
}
// LoadALL adds missing modules.
// Remove unused modules.
used := make(map[module.Version]bool)
for _, pkg := range modload.LoadALL() {
used[modload.PackageModule(pkg)] = true
}
used[modload.Target] = true // note: LoadALL initializes Target
inGoMod := make(map[string]bool)
for _, r := range modload.ModFile().Require {
inGoMod[r.Mod.Path] = true
}
var keep []module.Version
for _, m := range modload.BuildList() {
if used[m] {
keep = append(keep, m)
} else if cfg.BuildV && inGoMod[m.Path] {
fmt.Fprintf(os.Stderr, "unused %s\n", m.Path)
}
}
modload.SetBuildList(keep)
modload.LoadALL()
modload.TidyBuildList()
modTidyGoSum() // updates memory copy; WriteGoMod on next line flushes it out
modload.WriteGoMod()
}

78
libgo/go/cmd/go/internal/modcmd/vendor.go
View file

@ -18,7 +18,10 @@ import (
"cmd/go/internal/cfg"
"cmd/go/internal/imports"
"cmd/go/internal/modload"
"cmd/go/internal/module"
"cmd/go/internal/work"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
)
var cmdVendor = &base.Command{
@ -37,6 +40,7 @@ modules and packages to standard error.
func init() {
cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "")
work.AddModCommonFlags(cmdVendor)
}
func runVendor(cmd *base.Command, args []string) {
@ -59,19 +63,31 @@ func runVendor(cmd *base.Command, args []string) {
modpkgs[m] = append(modpkgs[m], pkg)
}
includeAllReplacements := false
isExplicit := map[module.Version]bool{}
if gv := modload.ModFile().Go; gv != nil && semver.Compare("v"+gv.Version, "v1.14") >= 0 {
// If the Go version is at least 1.14, annotate all explicit 'require' and
// 'replace' targets found in the go.mod file so that we can perform a
// stronger consistency check when -mod=vendor is set.
for _, r := range modload.ModFile().Require {
isExplicit[r.Mod] = true
}
includeAllReplacements = true
}
var buf bytes.Buffer
for _, m := range modload.BuildList()[1:] {
if pkgs := modpkgs[m]; len(pkgs) > 0 {
repl := ""
if r := modload.Replacement(m); r.Path != "" {
repl = " => " + r.Path
if r.Version != "" {
repl += " " + r.Version
}
}
fmt.Fprintf(&buf, "# %s %s%s\n", m.Path, m.Version, repl)
if pkgs := modpkgs[m]; len(pkgs) > 0 || isExplicit[m] {
line := moduleLine(m, modload.Replacement(m))
buf.WriteString(line)
if cfg.BuildV {
fmt.Fprintf(os.Stderr, "# %s %s%s\n", m.Path, m.Version, repl)
os.Stderr.WriteString(line)
}
if isExplicit[m] {
buf.WriteString("## explicit\n")
if cfg.BuildV {
os.Stderr.WriteString("## explicit\n")
}
}
sort.Strings(pkgs)
for _, pkg := range pkgs {
@ -83,6 +99,26 @@ func runVendor(cmd *base.Command, args []string) {
}
}
}
if includeAllReplacements {
// Record unused and wildcard replacements at the end of the modules.txt file:
// without access to the complete build list, the consumer of the vendor
// directory can't otherwise determine that those replacements had no effect.
for _, r := range modload.ModFile().Replace {
if len(modpkgs[r.Old]) > 0 {
// We we already recorded this replacement in the entry for the replaced
// module with the packages it provides.
continue
}
line := moduleLine(r.Old, r.New)
buf.WriteString(line)
if cfg.BuildV {
os.Stderr.WriteString(line)
}
}
}
if buf.Len() == 0 {
fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
return
@ -92,6 +128,26 @@ func runVendor(cmd *base.Command, args []string) {
}
}
func moduleLine(m, r module.Version) string {
b := new(strings.Builder)
b.WriteString("# ")
b.WriteString(m.Path)
if m.Version != "" {
b.WriteString(" ")
b.WriteString(m.Version)
}
if r.Path != "" {
b.WriteString(" => ")
b.WriteString(r.Path)
if r.Version != "" {
b.WriteString(" ")
b.WriteString(r.Version)
}
}
b.WriteString("\n")
return b.String()
}
func vendorPkg(vdir, pkg string) {
realPath := modload.ImportMap(pkg)
if realPath != pkg && modload.ImportMap(realPath) != "" {

14
libgo/go/cmd/go/internal/modcmd/verify.go
View file

@ -6,16 +6,18 @@ package modcmd
import (
"bytes"
"cmd/go/internal/cfg"
"fmt"
"io/ioutil"
"os"
"cmd/go/internal/base"
"cmd/go/internal/dirhash"
"cmd/go/internal/cfg"
"cmd/go/internal/modfetch"
"cmd/go/internal/modload"
"cmd/go/internal/module"
"cmd/go/internal/work"
"golang.org/x/mod/module"
"golang.org/x/mod/sumdb/dirhash"
)
var cmdVerify = &base.Command{
@ -32,13 +34,17 @@ non-zero status.
Run: runVerify,
}
func init() {
work.AddModCommonFlags(cmdVerify)
}
func runVerify(cmd *base.Command, args []string) {
if len(args) != 0 {
// NOTE(rsc): Could take a module pattern.
base.Fatalf("go mod verify: verify takes no arguments")
}
// Checks go mod expected behavior
if !modload.Enabled() {
if !modload.Enabled() || !modload.HasModRoot() {
if cfg.Getenv("GO111MODULE") == "off" {
base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'")
} else {

10
libgo/go/cmd/go/internal/modcmd/why.go
View file

@ -5,11 +5,14 @@
package modcmd
import (
"cmd/go/internal/base"
"cmd/go/internal/modload"
"cmd/go/internal/module"
"fmt"
"strings"
"cmd/go/internal/base"
"cmd/go/internal/modload"
"cmd/go/internal/work"
"golang.org/x/mod/module"
)
var cmdWhy = &base.Command{
@ -54,6 +57,7 @@ var (
func init() {
cmdWhy.Run = runWhy // break init cycle
work.AddModCommonFlags(cmdWhy)
}
func runWhy(cmd *base.Command, args []string) {

7
libgo/go/cmd/go/internal/modconv/convert.go
View file

@ -13,10 +13,11 @@ import (
"cmd/go/internal/base"
"cmd/go/internal/modfetch"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"cmd/go/internal/par"
"cmd/go/internal/semver"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
)
// ConvertLegacyConfig converts legacy config to modfile.

5
libgo/go/cmd/go/internal/modconv/convert_test.go
View file

@ -19,8 +19,9 @@ import (
"cmd/go/internal/cfg"
"cmd/go/internal/modfetch"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
func TestMain(m *testing.M) {

6
libgo/go/cmd/go/internal/modconv/dep.go
View file

@ -12,9 +12,9 @@ import (
"strconv"
"strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"cmd/go/internal/semver"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
)
func ParseGopkgLock(file string, data []byte) (*modfile.File, error) {

4
libgo/go/cmd/go/internal/modconv/glide.go
View file

@ -7,8 +7,8 @@ package modconv
import (
"strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
func ParseGlideLock(file string, data []byte) (*modfile.File, error) {

4
libgo/go/cmd/go/internal/modconv/glock.go
View file

@ -7,8 +7,8 @@ package modconv
import (
"strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
func ParseGLOCKFILE(file string, data []byte) (*modfile.File, error) {

4
libgo/go/cmd/go/internal/modconv/godeps.go
View file

@ -7,8 +7,8 @@ package modconv
import (
"encoding/json"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
func ParseGodepsJSON(file string, data []byte) (*modfile.File, error) {

2
libgo/go/cmd/go/internal/modconv/modconv.go
View file

@ -4,7 +4,7 @@
package modconv
import "cmd/go/internal/modfile"
import "golang.org/x/mod/modfile"
var Converters = map[string]func(string, []byte) (*modfile.File, error){
"GLOCKFILE": ParseGLOCKFILE,

4
libgo/go/cmd/go/internal/modconv/tsv.go
View file

@ -7,8 +7,8 @@ package modconv
import (
"strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
func ParseDependenciesTSV(file string, data []byte) (*modfile.File, error) {

4
libgo/go/cmd/go/internal/modconv/vconf.go
View file

@ -7,8 +7,8 @@ package modconv
import (
"strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
func ParseVendorConf(file string, data []byte) (*modfile.File, error) {

4
libgo/go/cmd/go/internal/modconv/vjson.go
View file

@ -7,8 +7,8 @@ package modconv
import (
"encoding/json"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
func ParseVendorJSON(file string, data []byte) (*modfile.File, error) {

4
libgo/go/cmd/go/internal/modconv/vmanifest.go
View file

@ -7,8 +7,8 @@ package modconv
import (
"encoding/json"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
func ParseVendorManifest(file string, data []byte) (*modfile.File, error) {

4
libgo/go/cmd/go/internal/modconv/vyml.go
View file

@ -7,8 +7,8 @@ package modconv
import (
"strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
func ParseVendorYML(file string, data []byte) (*modfile.File, error) {

2
libgo/go/cmd/go/internal/modfetch/bootstrap.go
View file

@ -6,7 +6,7 @@
package modfetch
import "cmd/go/internal/module"
import "golang.org/x/mod/module"
func useSumDB(mod module.Version) bool {
return false

62
libgo/go/cmd/go/internal/modfetch/cache.go
View file

@ -13,26 +13,28 @@ import (
"os"
"path/filepath"
"strings"
"time"
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/lockedfile"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/module"
"cmd/go/internal/par"
"cmd/go/internal/renameio"
"cmd/go/internal/semver"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
)
var QuietLookup bool // do not print about lookups
var PkgMod string // $GOPATH/pkg/mod; set by package modload
const logFindingDelay = 1 * time.Second
func cacheDir(path string) (string, error) {
if PkgMod == "" {
return "", fmt.Errorf("internal error: modfetch.PkgMod not set")
}
enc, err := module.EncodePath(path)
enc, err := module.EscapePath(path)
if err != nil {
return "", err
}
@ -50,7 +52,7 @@ func CachePath(m module.Version, suffix string) (string, error) {
if module.CanonicalVersion(m.Version) != m.Version {
return "", fmt.Errorf("non-canonical module version %q", m.Version)
}
encVer, err := module.EncodeVersion(m.Version)
encVer, err := module.EscapeVersion(m.Version)
if err != nil {
return "", err
}
@ -63,7 +65,7 @@ func DownloadDir(m module.Version) (string, error) {
if PkgMod == "" {
return "", fmt.Errorf("internal error: modfetch.PkgMod not set")
}
enc, err := module.EncodePath(m.Path)
enc, err := module.EscapePath(m.Path)
if err != nil {
return "", err
}
@ -73,7 +75,7 @@ func DownloadDir(m module.Version) (string, error) {
if module.CanonicalVersion(m.Version) != m.Version {
return "", fmt.Errorf("non-canonical module version %q", m.Version)
}
encVer, err := module.EncodeVersion(m.Version)
encVer, err := module.EscapeVersion(m.Version)
if err != nil {
return "", err
}
@ -93,22 +95,21 @@ func lockVersion(mod module.Version) (unlock func(), err error) {
return lockedfile.MutexAt(path).Lock()
}
// SideLock locks a file within the module cache that that guards edits to files
// outside the cache, such as go.sum and go.mod files in the user's working
// directory. It returns a function that must be called to unlock the file.
func SideLock() (unlock func()) {
// SideLock locks a file within the module cache that that previously guarded
// edits to files outside the cache, such as go.sum and go.mod files in the
// user's working directory.
// If err is nil, the caller MUST eventually call the unlock function.
func SideLock() (unlock func(), err error) {
if PkgMod == "" {
base.Fatalf("go: internal error: modfetch.PkgMod not set")
}
path := filepath.Join(PkgMod, "cache", "lock")
if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
base.Fatalf("go: failed to create cache directory %s: %v", filepath.Dir(path), err)
return nil, fmt.Errorf("failed to create cache directory: %w", err)
}
unlock, err := lockedfile.MutexAt(path).Lock()
if err != nil {
base.Fatalf("go: failed to lock file at %v", path)
}
return unlock
return lockedfile.MutexAt(path).Lock()
}
// A cachingRepo is a cache around an underlying Repo,
@ -139,6 +140,11 @@ func (r *cachingRepo) Versions(prefix string) ([]string, error) {
err error
}
c := r.cache.Do("versions:"+prefix, func() interface{} {
logTimer := time.AfterFunc(logFindingDelay, func() {
fmt.Fprintf(os.Stderr, "go: finding versions for %s\n", r.path)
})
defer logTimer.Stop()
list, err := r.r.Versions(prefix)
return cached{list, err}
}).(cached)
@ -161,9 +167,11 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) {
return cachedInfo{info, nil}
}
if !QuietLookup {
logTimer := time.AfterFunc(logFindingDelay, func() {
fmt.Fprintf(os.Stderr, "go: finding %s %s\n", r.path, rev)
}
})
defer logTimer.Stop()
info, err = r.r.Stat(rev)
if err == nil {
// If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78,
@ -191,9 +199,11 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) {
func (r *cachingRepo) Latest() (*RevInfo, error) {
c := r.cache.Do("latest:", func() interface{} {
if !QuietLookup {
logTimer := time.AfterFunc(logFindingDelay, func() {
fmt.Fprintf(os.Stderr, "go: finding %s latest\n", r.path)
}
})
defer logTimer.Stop()
info, err := r.r.Latest()
// Save info for likely future Stat call.
@ -230,7 +240,9 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) {
text, err = r.r.GoMod(version)
if err == nil {
checkGoMod(r.path, version, text)
if err := checkGoMod(r.path, version, text); err != nil {
return cached{text, err}
}
if err := writeDiskGoMod(file, text); err != nil {
fmt.Fprintf(os.Stderr, "go: writing go.mod cache: %v\n", err)
}
@ -490,7 +502,9 @@ func readDiskGoMod(path, rev string) (file string, data []byte, err error) {
}
if err == nil {
checkGoMod(path, rev, data)
if err := checkGoMod(path, rev, data); err != nil {
return "", nil, err
}
}
return file, data, err

7
libgo/go/cmd/go/internal/modfetch/codehost/codehost.go
View file

@ -73,11 +73,10 @@ type Repo interface {
// ReadZip downloads a zip file for the subdir subdirectory
// of the given revision to a new file in a given temporary directory.
// It should refuse to read more than maxSize bytes.
// It returns a ReadCloser for a streamed copy of the zip file,
// along with the actual subdirectory (possibly shorter than subdir)
// contained in the zip file. All files in the zip file are expected to be
// It returns a ReadCloser for a streamed copy of the zip file.
// All files in the zip file are expected to be
// nested in a single top-level directory, whose name is not specified.
ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error)
ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error)
// RecentTag returns the most recent tag on rev or one of its predecessors
// with the given prefix and major version.

125
libgo/go/cmd/go/internal/modfetch/codehost/git.go
View file

@ -6,9 +6,11 @@ package codehost
import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"os/exec"
"path/filepath"
@ -20,7 +22,9 @@ import (
"cmd/go/internal/lockedfile"
"cmd/go/internal/par"
"cmd/go/internal/semver"
"cmd/go/internal/web"
"golang.org/x/mod/semver"
)
// GitRepo returns the code repository at the given Git remote reference.
@ -34,6 +38,15 @@ func LocalGitRepo(remote string) (Repo, error) {
return newGitRepoCached(remote, true)
}
// A notExistError wraps another error to retain its original text
// but makes it opaquely equivalent to os.ErrNotExist.
type notExistError struct {
err error
}
func (e notExistError) Error() string { return e.err.Error() }
func (notExistError) Is(err error) bool { return err == os.ErrNotExist }
const gitWorkDirType = "git3"
var gitRepoCache par.Cache
@ -85,8 +98,9 @@ func newGitRepo(remote string, localOK bool) (Repo, error) {
os.RemoveAll(r.dir)
return nil, err
}
r.remote = "origin"
}
r.remoteURL = r.remote
r.remote = "origin"
} else {
// Local path.
// Disallow colon (not in ://) because sometimes
@ -113,9 +127,9 @@ func newGitRepo(remote string, localOK bool) (Repo, error) {
}
type gitRepo struct {
remote string
local bool
dir string
remote, remoteURL string
local bool
dir string
mu lockedfile.Mutex // protects fetchLevel and git repo state
@ -166,14 +180,25 @@ func (r *gitRepo) loadRefs() {
// The git protocol sends all known refs and ls-remote filters them on the client side,
// so we might as well record both heads and tags in one shot.
// Most of the time we only care about tags but sometimes we care about heads too.
out, err := Run(r.dir, "git", "ls-remote", "-q", r.remote)
if err != nil {
if rerr, ok := err.(*RunError); ok {
out, gitErr := Run(r.dir, "git", "ls-remote", "-q", r.remote)
if gitErr != nil {
if rerr, ok := gitErr.(*RunError); ok {
if bytes.Contains(rerr.Stderr, []byte("fatal: could not read Username")) {
rerr.HelpText = "Confirm the import path was entered correctly.\nIf this is a private repository, see https://golang.org/doc/faq#git_https for additional information."
}
}
r.refsErr = err
// If the remote URL doesn't exist at all, ideally we should treat the whole
// repository as nonexistent by wrapping the error in a notExistError.
// For HTTP and HTTPS, that's easy to detect: we'll try to fetch the URL
// ourselves and see what code it serves.
if u, err := url.Parse(r.remoteURL); err == nil && (u.Scheme == "http" || u.Scheme == "https") {
if _, err := web.GetBytes(u); errors.Is(err, os.ErrNotExist) {
gitErr = notExistError{gitErr}
}
}
r.refsErr = gitErr
return
}
@ -241,13 +266,6 @@ func (r *gitRepo) findRef(hash string) (ref string, ok bool) {
return "", false
}
func unshallow(gitDir string) []string {
if _, err := os.Stat(filepath.Join(gitDir, "shallow")); err == nil {
return []string{"--unshallow"}
}
return []string{}
}
// minHashDigits is the minimum number of digits to require
// before accepting a hex digit sequence as potentially identifying
// a specific commit in a git repo. (Of course, users can always
@ -397,29 +415,27 @@ func (r *gitRepo) stat(rev string) (*RevInfo, error) {
// fetchRefsLocked requires that r.mu remain locked for the duration of the call.
func (r *gitRepo) fetchRefsLocked() error {
if r.fetchLevel < fetchAll {
if err := r.fetchUnshallow("refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"); err != nil {
// NOTE: To work around a bug affecting Git clients up to at least 2.23.0
// (2019-08-16), we must first expand the set of local refs, and only then
// unshallow the repository as a separate fetch operation. (See
// golang.org/issue/34266 and
// https://github.com/git/git/blob/4c86140027f4a0d2caaa3ab4bd8bfc5ce3c11c8a/transport.c#L1303-L1309.)
if _, err := Run(r.dir, "git", "fetch", "-f", r.remote, "refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"); err != nil {
return err
}
if _, err := os.Stat(filepath.Join(r.dir, "shallow")); err == nil {
if _, err := Run(r.dir, "git", "fetch", "--unshallow", "-f", r.remote); err != nil {
return err
}
}
r.fetchLevel = fetchAll
}
return nil
}
func (r *gitRepo) fetchUnshallow(refSpecs ...string) error {
// To work around a protocol version 2 bug that breaks --unshallow,
// add -c protocol.version=0.
// TODO(rsc): The bug is believed to be server-side, meaning only
// on Google's Git servers. Once the servers are fixed, drop the
// protocol.version=0. See Google-internal bug b/110495752.
var protoFlag []string
unshallowFlag := unshallow(r.dir)
if len(unshallowFlag) > 0 {
protoFlag = []string{"-c", "protocol.version=0"}
}
_, err := Run(r.dir, "git", protoFlag, "fetch", unshallowFlag, "-f", r.remote, refSpecs)
return err
}
// statLocal returns a RevInfo describing rev in the local git repository.
// It uses version as info.Version.
func (r *gitRepo) statLocal(version, rev string) (*RevInfo, error) {
@ -539,39 +555,10 @@ func (r *gitRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[s
}
defer unlock()
var refs []string
var protoFlag []string
var unshallowFlag []string
for _, tag := range redo {
refs = append(refs, "refs/tags/"+tag+":refs/tags/"+tag)
}
if len(refs) > 1 {
unshallowFlag = unshallow(r.dir)
if len(unshallowFlag) > 0 {
// To work around a protocol version 2 bug that breaks --unshallow,
// add -c protocol.version=0.
// TODO(rsc): The bug is believed to be server-side, meaning only
// on Google's Git servers. Once the servers are fixed, drop the
// protocol.version=0. See Google-internal bug b/110495752.
protoFlag = []string{"-c", "protocol.version=0"}
}
}
if _, err := Run(r.dir, "git", protoFlag, "fetch", unshallowFlag, "-f", r.remote, refs); err != nil {
if err := r.fetchRefsLocked(); err != nil {
return nil, err
}
// TODO(bcmills): after the 1.11 freeze, replace the block above with:
// if r.fetchLevel <= fetchSome {
// r.fetchLevel = fetchSome
// var refs []string
// for _, tag := range redo {
// refs = append(refs, "refs/tags/"+tag+":refs/tags/"+tag)
// }
// if _, err := Run(r.dir, "git", "fetch", "--update-shallow", "-f", r.remote, refs); err != nil {
// return nil, err
// }
// }
if _, err := r.readFileRevs(redo, file, files); err != nil {
return nil, err
}
@ -809,7 +796,7 @@ func (r *gitRepo) DescendsFrom(rev, tag string) (bool, error) {
return false, err
}
func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) {
func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) {
// TODO: Use maxSize or drop it.
args := []string{}
if subdir != "" {
@ -817,17 +804,17 @@ func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser,
}
info, err := r.Stat(rev) // download rev into local git repo
if err != nil {
return nil, "", err
return nil, err
}
unlock, err := r.mu.Lock()
if err != nil {
return nil, "", err
return nil, err
}
defer unlock()
if err := ensureGitAttributes(r.dir); err != nil {
return nil, "", err
return nil, err
}
// Incredibly, git produces different archives depending on whether
@ -838,12 +825,12 @@ func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser,
archive, err := Run(r.dir, "git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", "--prefix=prefix/", info.Name, args)
if err != nil {
if bytes.Contains(err.(*RunError).Stderr, []byte("did not match any files")) {
return nil, "", os.ErrNotExist
return nil, os.ErrNotExist
}
return nil, "", err
return nil, err
}
return ioutil.NopCloser(bytes.NewReader(archive)), "", nil
return ioutil.NopCloser(bytes.NewReader(archive)), nil
}
// ensureGitAttributes makes sure export-subst and export-ignore features are

27
libgo/go/cmd/go/internal/modfetch/codehost/git_test.go
View file

@ -78,7 +78,16 @@ func testMain(m *testing.M) int {
func testRepo(remote string) (Repo, error) {
if remote == "localGitRepo" {
return LocalGitRepo(filepath.ToSlash(localGitRepo))
// Convert absolute path to file URL. LocalGitRepo will not accept
// Windows absolute paths because they look like a host:path remote.
// TODO(golang.org/issue/32456): use url.FromFilePath when implemented.
var url string
if strings.HasPrefix(localGitRepo, "/") {
url = "file://" + localGitRepo
} else {
url = "file:///" + filepath.ToSlash(localGitRepo)
}
return LocalGitRepo(url)
}
kind := "git"
for _, k := range []string{"hg"} {
@ -246,12 +255,11 @@ func TestReadFile(t *testing.T) {
}
var readZipTests = []struct {
repo string
rev string
subdir string
actualSubdir string
err string
files map[string]uint64
repo string
rev string
subdir string
err string
files map[string]uint64
}{
{
repo: gitrepo1,
@ -408,7 +416,7 @@ func TestReadZip(t *testing.T) {
if err != nil {
t.Fatal(err)
}
rc, actualSubdir, err := r.ReadZip(tt.rev, tt.subdir, 100000)
rc, err := r.ReadZip(tt.rev, tt.subdir, 100000)
if err != nil {
if tt.err == "" {
t.Fatalf("ReadZip: unexpected error %v", err)
@ -425,9 +433,6 @@ func TestReadZip(t *testing.T) {
if tt.err != "" {
t.Fatalf("ReadZip: no error, wanted %v", tt.err)
}
if actualSubdir != tt.actualSubdir {
t.Fatalf("ReadZip: actualSubdir = %q, want %q", actualSubdir, tt.actualSubdir)
}
zipdata, err := ioutil.ReadAll(rc)
if err != nil {
t.Fatal(err)

2
libgo/go/cmd/go/internal/modfetch/codehost/shell.go
View file

@ -109,7 +109,7 @@ func main() {
if subdir == "-" {
subdir = ""
}
rc, _, err := repo.ReadZip(f[1], subdir, 10<<20)
rc, err := repo.ReadZip(f[1], subdir, 10<<20)
if err != nil {
fmt.Fprintf(os.Stderr, "?%s\n", err)
continue

154
libgo/go/cmd/go/internal/modfetch/codehost/svn.go Normal file
View file

@ -0,0 +1,154 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package codehost
import (
"archive/zip"
"encoding/xml"
"fmt"
"io"
"os"
"path"
"path/filepath"
"time"
)
func svnParseStat(rev, out string) (*RevInfo, error) {
var log struct {
Logentry struct {
Revision int64 `xml:"revision,attr"`
Date string `xml:"date"`
} `xml:"logentry"`
}
if err := xml.Unmarshal([]byte(out), &log); err != nil {
return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out)
}
t, err := time.Parse(time.RFC3339, log.Logentry.Date)
if err != nil {
return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out)
}
info := &RevInfo{
Name: fmt.Sprintf("%d", log.Logentry.Revision),
Short: fmt.Sprintf("%012d", log.Logentry.Revision),
Time: t.UTC(),
Version: rev,
}
return info, nil
}
func svnReadZip(dst io.Writer, workDir, rev, subdir, remote string) (err error) {
// The subversion CLI doesn't provide a command to write the repository
// directly to an archive, so we need to export it to the local filesystem
// instead. Unfortunately, the local filesystem might apply arbitrary
// normalization to the filenames, so we need to obtain those directly.
//
// 'svn export' prints the filenames as they are written, but from reading the
// svn source code (as of revision 1868933), those filenames are encoded using
// the system locale rather than preserved byte-for-byte from the origin. For
// our purposes, that won't do, but we don't want to go mucking around with
// the user's locale settings either — that could impact error messages, and
// we don't know what locales the user has available or what LC_* variables
// their platform supports.
//
// Instead, we'll do a two-pass export: first we'll run 'svn list' to get the
// canonical filenames, then we'll 'svn export' and look for those filenames
// in the local filesystem. (If there is an encoding problem at that point, we
// would probably reject the resulting module anyway.)
remotePath := remote
if subdir != "" {
remotePath += "/" + subdir
}
out, err := Run(workDir, []string{
"svn", "list",
"--non-interactive",
"--xml",
"--incremental",
"--recursive",
"--revision", rev,
"--", remotePath,
})
if err != nil {
return err
}
type listEntry struct {
Kind string `xml:"kind,attr"`
Name string `xml:"name"`
Size int64 `xml:"size"`
}
var list struct {
Entries []listEntry `xml:"entry"`
}
if err := xml.Unmarshal(out, &list); err != nil {
return vcsErrorf("unexpected response from svn list --xml: %v\n%s", err, out)
}
exportDir := filepath.Join(workDir, "export")
// Remove any existing contents from a previous (failed) run.
if err := os.RemoveAll(exportDir); err != nil {
return err
}
defer os.RemoveAll(exportDir) // best-effort
_, err = Run(workDir, []string{
"svn", "export",
"--non-interactive",
"--quiet",
// Suppress any platform- or host-dependent transformations.
"--native-eol", "LF",
"--ignore-externals",
"--ignore-keywords",
"--revision", rev,
"--", remotePath,
exportDir,
})
if err != nil {
return err
}
// Scrape the exported files out of the filesystem and encode them in the zipfile.
// “All files in the zip file are expected to be
// nested in a single top-level directory, whose name is not specified.”
// We'll (arbitrarily) choose the base of the remote path.
basePath := path.Join(path.Base(remote), subdir)
zw := zip.NewWriter(dst)
for _, e := range list.Entries {
if e.Kind != "file" {
continue
}
zf, err := zw.Create(path.Join(basePath, e.Name))
if err != nil {
return err
}
f, err := os.Open(filepath.Join(exportDir, e.Name))
if err != nil {
if os.IsNotExist(err) {
return vcsErrorf("file reported by 'svn list', but not written by 'svn export': %s", e.Name)
}
return fmt.Errorf("error opening file created by 'svn export': %v", err)
}
n, err := io.Copy(zf, f)
f.Close()
if err != nil {
return err
}
if n != e.Size {
return vcsErrorf("file size differs between 'svn list' and 'svn export': file %s listed as %v bytes, but exported as %v bytes", e.Name, e.Size, n)
}
}
return zw.Close()
}

104
libgo/go/cmd/go/internal/modfetch/codehost/vcs.go
View file

@ -5,7 +5,7 @@
package codehost
import (
"encoding/xml"
"errors"
"fmt"
"internal/lazyregexp"
"io"
@ -122,19 +122,20 @@ func newVCSRepo(vcs, remote string) (Repo, error) {
const vcsWorkDirType = "vcs1."
type vcsCmd struct {
vcs string // vcs name "hg"
init func(remote string) []string // cmd to init repo to track remote
tags func(remote string) []string // cmd to list local tags
tagRE *lazyregexp.Regexp // regexp to extract tag names from output of tags cmd
branches func(remote string) []string // cmd to list local branches
branchRE *lazyregexp.Regexp // regexp to extract branch names from output of tags cmd
badLocalRevRE *lazyregexp.Regexp // regexp of names that must not be served out of local cache without doing fetch first
statLocal func(rev, remote string) []string // cmd to stat local rev
parseStat func(rev, out string) (*RevInfo, error) // cmd to parse output of statLocal
fetch []string // cmd to fetch everything from remote
latest string // name of latest commit on remote (tip, HEAD, etc)
readFile func(rev, file, remote string) []string // cmd to read rev's file
readZip func(rev, subdir, remote, target string) []string // cmd to read rev's subdir as zip file
vcs string // vcs name "hg"
init func(remote string) []string // cmd to init repo to track remote
tags func(remote string) []string // cmd to list local tags
tagRE *lazyregexp.Regexp // regexp to extract tag names from output of tags cmd
branches func(remote string) []string // cmd to list local branches
branchRE *lazyregexp.Regexp // regexp to extract branch names from output of tags cmd
badLocalRevRE *lazyregexp.Regexp // regexp of names that must not be served out of local cache without doing fetch first
statLocal func(rev, remote string) []string // cmd to stat local rev
parseStat func(rev, out string) (*RevInfo, error) // cmd to parse output of statLocal
fetch []string // cmd to fetch everything from remote
latest string // name of latest commit on remote (tip, HEAD, etc)
readFile func(rev, file, remote string) []string // cmd to read rev's file
readZip func(rev, subdir, remote, target string) []string // cmd to read rev's subdir as zip file
doReadZip func(dst io.Writer, workDir, rev, subdir, remote string) error // arbitrary function to read rev's subdir as zip file
}
var re = lazyregexp.New
@ -191,7 +192,7 @@ var vcsCmds = map[string]*vcsCmd{
readFile: func(rev, file, remote string) []string {
return []string{"svn", "cat", "--", remote + "/" + file + "@" + rev}
},
// TODO: zip
doReadZip: svnReadZip,
},
"bzr": {
@ -417,14 +418,14 @@ func (r *vcsRepo) DescendsFrom(rev, tag string) (bool, error) {
return false, vcsErrorf("DescendsFrom not implemented")
}
func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) {
if r.cmd.readZip == nil {
return nil, "", vcsErrorf("ReadZip not implemented for %s", r.cmd.vcs)
func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) {
if r.cmd.readZip == nil && r.cmd.doReadZip == nil {
return nil, vcsErrorf("ReadZip not implemented for %s", r.cmd.vcs)
}
unlock, err := r.mu.Lock()
if err != nil {
return nil, "", err
return nil, err
}
defer unlock()
@ -433,9 +434,19 @@ func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser,
}
f, err := ioutil.TempFile("", "go-readzip-*.zip")
if err != nil {
return nil, "", err
return nil, err
}
if r.cmd.vcs == "fossil" {
if r.cmd.doReadZip != nil {
lw := &limitedWriter{
W: f,
N: maxSize,
ErrLimitReached: errors.New("ReadZip: encoded file exceeds allowed size"),
}
err = r.cmd.doReadZip(lw, r.dir, rev, subdir, r.remote)
if err == nil {
_, err = f.Seek(0, io.SeekStart)
}
} else if r.cmd.vcs == "fossil" {
// If you run
// fossil zip -R .fossil --name prefix trunk /tmp/x.zip
// fossil fails with "unable to create directory /tmp" [sic].
@ -454,9 +465,9 @@ func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser,
if err != nil {
f.Close()
os.Remove(f.Name())
return nil, "", err
return nil, err
}
return &deleteCloser{f}, "", nil
return &deleteCloser{f}, nil
}
// deleteCloser is a file that gets deleted on Close.
@ -502,31 +513,6 @@ func hgParseStat(rev, out string) (*RevInfo, error) {
return info, nil
}
func svnParseStat(rev, out string) (*RevInfo, error) {
var log struct {
Logentry struct {
Revision int64 `xml:"revision,attr"`
Date string `xml:"date"`
} `xml:"logentry"`
}
if err := xml.Unmarshal([]byte(out), &log); err != nil {
return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out)
}
t, err := time.Parse(time.RFC3339, log.Logentry.Date)
if err != nil {
return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out)
}
info := &RevInfo{
Name: fmt.Sprintf("%d", log.Logentry.Revision),
Short: fmt.Sprintf("%012d", log.Logentry.Revision),
Time: t.UTC(),
Version: rev,
}
return info, nil
}
func bzrParseStat(rev, out string) (*RevInfo, error) {
var revno int64
var tm time.Time
@ -606,3 +592,25 @@ func fossilParseStat(rev, out string) (*RevInfo, error) {
}
return nil, vcsErrorf("unexpected response from fossil info: %q", out)
}
type limitedWriter struct {
W io.Writer
N int64
ErrLimitReached error
}
func (l *limitedWriter) Write(p []byte) (n int, err error) {
if l.N > 0 {
max := len(p)
if l.N < int64(max) {
max = int(l.N)
}
n, err = l.W.Write(p[:max])
l.N -= int64(n)
if err != nil || n >= len(p) {
return n, err
}
}
return n, l.ErrLimitReached
}

310
libgo/go/cmd/go/internal/modfetch/coderepo.go
View file

@ -6,19 +6,23 @@ package modfetch
import (
"archive/zip"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"sort"
"strings"
"time"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
"cmd/go/internal/semver"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
modzip "golang.org/x/mod/zip"
)
// A codeRepo implements modfetch.Repo using an underlying codehost.Repo.
@ -140,11 +144,13 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) {
}
tags, err := r.code.Tags(p)
if err != nil {
return nil, err
return nil, &module.ModuleError{
Path: r.modPath,
Err: err,
}
}
list := []string{}
var incompatible []string
var list, incompatible []string
for _, tag := range tags {
if !strings.HasPrefix(tag, p) {
continue
@ -156,32 +162,114 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) {
if v == "" || v != module.CanonicalVersion(v) || IsPseudoVersion(v) {
continue
}
if err := module.MatchPathMajor(v, r.pathMajor); err != nil {
if err := module.CheckPathMajor(v, r.pathMajor); err != nil {
if r.codeDir == "" && r.pathMajor == "" && semver.Major(v) > "v1" {
incompatible = append(incompatible, v)
}
continue
}
list = append(list, v)
}
SortVersions(list)
SortVersions(incompatible)
if len(incompatible) > 0 {
// Check for later versions that were created not following semantic import versioning,
// as indicated by the absence of a go.mod file. Those versions can be addressed
// by referring to them with a +incompatible suffix, as in v17.0.0+incompatible.
files, err := r.code.ReadFileRevs(incompatible, "go.mod", codehost.MaxGoMod)
return r.appendIncompatibleVersions(list, incompatible)
}
// appendIncompatibleVersions appends "+incompatible" versions to list if
// appropriate, returning the final list.
//
// The incompatible list contains candidate versions without the '+incompatible'
// prefix.
//
// Both list and incompatible must be sorted in semantic order.
func (r *codeRepo) appendIncompatibleVersions(list, incompatible []string) ([]string, error) {
if len(incompatible) == 0 || r.pathMajor != "" {
// No +incompatible versions are possible, so no need to check them.
return list, nil
}
// We assume that if the latest release of any major version has a go.mod
// file, all subsequent major versions will also have go.mod files (and thus
// be ineligible for use as +incompatible versions).
// If we're wrong about a major version, users will still be able to 'go get'
// specific higher versions explicitly — they just won't affect 'latest' or
// appear in 'go list'.
//
// Conversely, we assume that if the latest release of any major version lacks
// a go.mod file, all versions also lack go.mod files. If we're wrong, we may
// include a +incompatible version that isn't really valid, but most
// operations won't try to use that version anyway.
//
// These optimizations bring
// 'go list -versions -m github.com/openshift/origin' down from 1m58s to 0m37s.
// That's still not great, but a substantial improvement.
versionHasGoMod := func(v string) (bool, error) {
_, err := r.code.ReadFile(v, "go.mod", codehost.MaxGoMod)
if err == nil {
return true, nil
}
if !os.IsNotExist(err) {
return false, &module.ModuleError{
Path: r.modPath,
Err: err,
}
}
return false, nil
}
if len(list) > 0 {
ok, err := versionHasGoMod(list[len(list)-1])
if err != nil {
return nil, err
}
for _, rev := range incompatible {
f := files[rev]
if os.IsNotExist(f.Err) {
list = append(list, rev+"+incompatible")
}
if ok {
// The latest compatible version has a go.mod file, so assume that all
// subsequent versions do as well, and do not include any +incompatible
// versions. Even if we are wrong, the author clearly intends module
// consumers to be on the v0/v1 line instead of a higher +incompatible
// version. (See https://golang.org/issue/34189.)
//
// We know of at least two examples where this behavior is desired
// (github.com/russross/blackfriday@v2.0.0 and
// github.com/libp2p/go-libp2p@v6.0.23), and (as of 2019-10-29) have no
// concrete examples for which it is undesired.
return list, nil
}
}
SortVersions(list)
var lastMajor string
for i, v := range incompatible {
major := semver.Major(v)
if major == lastMajor {
list = append(list, v+"+incompatible")
continue
}
rem := incompatible[i:]
j := sort.Search(len(rem), func(j int) bool {
return semver.Major(rem[j]) != major
})
latestAtMajor := rem[j-1]
ok, err := versionHasGoMod(latestAtMajor)
if err != nil {
return nil, err
}
if ok {
// This major version has a go.mod file, so it is not allowed as
// +incompatible. Subsequent major versions are likely to also have
// go.mod files, so stop here.
break
}
lastMajor = major
list = append(list, v+"+incompatible")
}
return list, nil
}
@ -271,7 +359,7 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
Path: r.modPath,
Err: &module.InvalidVersionError{
Version: info2.Version,
Err: notExistError(err.Error()),
Err: notExistError{err: err},
},
}
}
@ -287,7 +375,7 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
}
}
if err := module.MatchPathMajor(strings.TrimSuffix(info2.Version, "+incompatible"), r.pathMajor); err == nil {
if err := module.CheckPathMajor(strings.TrimSuffix(info2.Version, "+incompatible"), r.pathMajor); err == nil {
return nil, invalidf("+incompatible suffix not allowed: major version %s is compatible", semver.Major(info2.Version))
}
}
@ -311,7 +399,7 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
return checkGoMod()
}
if err := module.MatchPathMajor(info2.Version, r.pathMajor); err != nil {
if err := module.CheckPathMajor(info2.Version, r.pathMajor); err != nil {
if canUseIncompatible() {
info2.Version += "+incompatible"
return checkGoMod()
@ -359,7 +447,7 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
tagIsCanonical = true
}
if err := module.MatchPathMajor(v, r.pathMajor); err != nil {
if err := module.CheckPathMajor(v, r.pathMajor); err != nil {
if canUseIncompatible() {
return v + "+incompatible", tagIsCanonical
}
@ -458,7 +546,7 @@ func (r *codeRepo) validatePseudoVersion(info *codehost.RevInfo, version string)
}
}()
if err := module.MatchPathMajor(version, r.pathMajor); err != nil {
if err := module.CheckPathMajor(version, r.pathMajor); err != nil {
return err
}
@ -631,9 +719,6 @@ func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err e
// because of replacement modules. This might be a fork of
// the real module, found at a different path, usable only in
// a replace directive.
//
// TODO(bcmills): This doesn't seem right. Investigate futher.
// (Notably: why can't we replace foo/v2 with fork-of-foo/v3?)
dir2 := path.Join(r.codeDir, r.pathMajor[1:])
file2 = path.Join(dir2, "go.mod")
gomod2, err2 := r.code.ReadFile(rev, file2, codehost.MaxGoMod)
@ -659,11 +744,11 @@ func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err e
// Not v2/go.mod, so it's either go.mod or nothing. Which is it?
if found1 {
// Explicit go.mod with matching module path OK.
// Explicit go.mod with matching major version ok.
return rev, r.codeDir, gomod1, nil
}
if err1 == nil {
// Explicit go.mod with non-matching module path disallowed.
// Explicit go.mod with non-matching major version disallowed.
suffix := ""
if file2 != "" {
suffix = fmt.Sprintf(" (and ...%s/go.mod does not exist)", r.pathMajor)
@ -674,6 +759,9 @@ func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err e
if r.pathMajor != "" { // ".v1", ".v2" for gopkg.in
return "", "", nil, fmt.Errorf("%s has non-...%s module path %q%s at revision %s", file1, r.pathMajor, mpath1, suffix, rev)
}
if _, _, ok := module.SplitPathVersion(mpath1); !ok {
return "", "", nil, fmt.Errorf("%s has malformed module path %q%s at revision %s", file1, mpath1, suffix, rev)
}
return "", "", nil, fmt.Errorf("%s has post-%s module path %q%s at revision %s", file1, semver.Major(version), mpath1, suffix, rev)
}
@ -690,24 +778,43 @@ func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err e
return "", "", nil, fmt.Errorf("missing %s/go.mod at revision %s", r.pathPrefix, rev)
}
// isMajor reports whether the versions allowed for mpath are compatible with
// the major version(s) implied by pathMajor, or false if mpath has an invalid
// version suffix.
func isMajor(mpath, pathMajor string) bool {
if mpath == "" {
// If we don't have a path, we don't know what version(s) it is compatible with.
return false
}
_, mpathMajor, ok := module.SplitPathVersion(mpath)
if !ok {
// An invalid module path is not compatible with any version.
return false
}
if pathMajor == "" {
// mpath must NOT have version suffix.
i := len(mpath)
for i > 0 && '0' <= mpath[i-1] && mpath[i-1] <= '9' {
i--
}
if i < len(mpath) && i >= 2 && mpath[i-1] == 'v' && mpath[i-2] == '/' {
// Found valid suffix.
// All of the valid versions for a gopkg.in module that requires major
// version v0 or v1 are compatible with the "v0 or v1" implied by an empty
// pathMajor.
switch module.PathMajorPrefix(mpathMajor) {
case "", "v0", "v1":
return true
default:
return false
}
return true
}
// Otherwise pathMajor is ".v1", ".v2" (gopkg.in), or "/v2", "/v3" etc.
return strings.HasSuffix(mpath, pathMajor)
if mpathMajor == "" {
// Even if pathMajor is ".v0" or ".v1", we can't be sure that a module
// without a suffix is tagged appropriately. Besides, we don't expect clones
// of non-gopkg.in modules to have gopkg.in paths, so a non-empty,
// non-gopkg.in mpath is probably the wrong module for any such pathMajor
// anyway.
return false
}
// If both pathMajor and mpathMajor are non-empty, then we only care that they
// have the same major-version validation rules. A clone fetched via a /v2
// path might replace a module with path gopkg.in/foo.v2-unstable, and that's
// ok.
return pathMajor[1:] == mpathMajor[1:]
}
func (r *codeRepo) GoMod(version string) (data []byte, err error) {
@ -774,19 +881,16 @@ func (r *codeRepo) Zip(dst io.Writer, version string) error {
}
}
rev, dir, _, err := r.findDir(version)
rev, subdir, _, err := r.findDir(version)
if err != nil {
return err
}
dl, actualDir, err := r.code.ReadZip(rev, dir, codehost.MaxZipFile)
dl, err := r.code.ReadZip(rev, subdir, codehost.MaxZipFile)
if err != nil {
return err
}
defer dl.Close()
if actualDir != "" && !hasPathPrefix(dir, actualDir) {
return fmt.Errorf("internal error: downloading %v %v: dir=%q but actualDir=%q", r.modPath, rev, dir, actualDir)
}
subdir := strings.Trim(strings.TrimPrefix(dir, actualDir), "/")
subdir = strings.Trim(subdir, "/")
// Spool to local file.
f, err := ioutil.TempFile("", "go-codehost-")
@ -817,13 +921,12 @@ func (r *codeRepo) Zip(dst io.Writer, version string) error {
return err
}
zw := zip.NewWriter(dst)
var files []modzip.File
if subdir != "" {
subdir += "/"
}
haveLICENSE := false
topPrefix := ""
haveGoMod := make(map[string]bool)
for _, zf := range zr.File {
if topPrefix == "" {
i := strings.Index(zf.Name, "/")
@ -835,106 +938,61 @@ func (r *codeRepo) Zip(dst io.Writer, version string) error {
if !strings.HasPrefix(zf.Name, topPrefix) {
return fmt.Errorf("zip file contains more than one top-level directory")
}
dir, file := path.Split(zf.Name)
if file == "go.mod" {
haveGoMod[dir] = true
}
}
root := topPrefix + subdir
inSubmodule := func(name string) bool {
for {
dir, _ := path.Split(name)
if len(dir) <= len(root) {
return false
}
if haveGoMod[dir] {
return true
}
name = dir[:len(dir)-1]
}
}
for _, zf := range zr.File {
if !zf.FileInfo().Mode().IsRegular() {
// Skip symlinks (golang.org/issue/27093).
continue
}
if topPrefix == "" {
i := strings.Index(zf.Name, "/")
if i < 0 {
return fmt.Errorf("missing top-level directory prefix")
}
topPrefix = zf.Name[:i+1]
}
if strings.HasSuffix(zf.Name, "/") { // drop directory dummy entries
continue
}
if !strings.HasPrefix(zf.Name, topPrefix) {
return fmt.Errorf("zip file contains more than one top-level directory")
}
name := strings.TrimPrefix(zf.Name, topPrefix)
if !strings.HasPrefix(name, subdir) {
continue
}
if name == ".hg_archival.txt" {
// Inserted by hg archive.
// Not correct to drop from other version control systems, but too bad.
continue
}
name = strings.TrimPrefix(name, subdir)
if isVendoredPackage(name) {
if name == "" || strings.HasSuffix(name, "/") {
continue
}
if inSubmodule(zf.Name) {
continue
}
base := path.Base(name)
if strings.ToLower(base) == "go.mod" && base != "go.mod" {
return fmt.Errorf("zip file contains %s, want all lower-case go.mod", zf.Name)
}
files = append(files, zipFile{name: name, f: zf})
if name == "LICENSE" {
haveLICENSE = true
}
size := int64(zf.UncompressedSize64)
if size < 0 || maxSize < size {
return fmt.Errorf("module source tree too big")
}
maxSize -= size
rc, err := zf.Open()
if err != nil {
return err
}
w, err := zw.Create(r.modPrefix(version) + "/" + name)
if err != nil {
return err
}
lr := &io.LimitedReader{R: rc, N: size + 1}
if _, err := io.Copy(w, lr); err != nil {
return err
}
if lr.N <= 0 {
return fmt.Errorf("individual file too large")
}
}
if !haveLICENSE && subdir != "" {
data, err := r.code.ReadFile(rev, "LICENSE", codehost.MaxLICENSE)
if err == nil {
w, err := zw.Create(r.modPrefix(version) + "/LICENSE")
if err != nil {
return err
}
if _, err := w.Write(data); err != nil {
return err
}
files = append(files, dataFile{name: "LICENSE", data: data})
}
}
return zw.Close()
return modzip.Create(dst, module.Version{Path: r.modPath, Version: version}, files)
}
type zipFile struct {
name string
f *zip.File
}
func (f zipFile) Path() string { return f.name }
func (f zipFile) Lstat() (os.FileInfo, error) { return f.f.FileInfo(), nil }
func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() }
type dataFile struct {
name string
data []byte
}
func (f dataFile) Path() string { return f.name }
func (f dataFile) Lstat() (os.FileInfo, error) { return dataFileInfo{f}, nil }
func (f dataFile) Open() (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewReader(f.data)), nil
}
type dataFileInfo struct {
f dataFile
}
func (fi dataFileInfo) Name() string { return path.Base(fi.f.name) }
func (fi dataFileInfo) Size() int64 { return int64(len(fi.f.data)) }
func (fi dataFileInfo) Mode() os.FileMode { return 0644 }
func (fi dataFileInfo) ModTime() time.Time { return time.Time{} }
func (fi dataFileInfo) IsDir() bool { return false }
func (fi dataFileInfo) Sys() interface{} { return nil }
// hasPathPrefix reports whether the path s begins with the
// elements in prefix.
func hasPathPrefix(s, prefix string) bool {

362
libgo/go/cmd/go/internal/modfetch/coderepo_test.go
View file

@ -6,7 +6,11 @@ package modfetch
import (
"archive/zip"
"crypto/sha256"
"encoding/hex"
"hash"
"internal/testenv"
"io"
"io/ioutil"
"log"
"os"
@ -17,6 +21,8 @@ import (
"cmd/go/internal/cfg"
"cmd/go/internal/modfetch/codehost"
"golang.org/x/mod/sumdb/dirhash"
)
func TestMain(m *testing.M) {
@ -52,20 +58,22 @@ var altVgotests = map[string]string{
}
type codeRepoTest struct {
vcs string
path string
lookerr string
mpath string
rev string
err string
version string
name string
short string
time time.Time
gomod string
gomoderr string
zip []string
ziperr string
vcs string
path string
lookErr string
mpath string
rev string
err string
version string
name string
short string
time time.Time
gomod string
gomodErr string
zip []string
zipErr string
zipSum string
zipFileHash string
}
var codeRepoTests = []codeRepoTest{
@ -82,6 +90,8 @@ var codeRepoTests = []codeRepoTest{
"README.md",
"pkg/p.go",
},
zipSum: "h1:zVEjciLdlk/TPWCOyZo7k24T+tOKRQC+u8MKq/xS80I=",
zipFileHash: "738a00ddbfe8c329dce6b48e1f23c8e22a92db50f3cfb2653caa0d62676bc09c",
},
{
vcs: "git",
@ -96,6 +106,8 @@ var codeRepoTests = []codeRepoTest{
"README.md",
"pkg/p.go",
},
zipSum: "h1:nOznk2xKsLGkTnXe0q9t1Ewt9jxK+oadtafSUqHM3Ec=",
zipFileHash: "bacb08f391e29d2eaaef8281b5c129ee6d890e608ee65877e0003c0181a766c8",
},
{
vcs: "git",
@ -116,6 +128,8 @@ var codeRepoTests = []codeRepoTest{
"README.md",
"pkg/p.go",
},
zipSum: "h1:e040hOoWGeuJLawDjK9DW6med+cz9FxMFYDMOVG8ctQ=",
zipFileHash: "74caab65cfbea427c341fa815f3bb0378681d8f0e3cf62a7f207014263ec7be3",
},
{
vcs: "git",
@ -140,6 +154,8 @@ var codeRepoTests = []codeRepoTest{
"README.md",
"pkg/p.go",
},
zipSum: "h1:e040hOoWGeuJLawDjK9DW6med+cz9FxMFYDMOVG8ctQ=",
zipFileHash: "74caab65cfbea427c341fa815f3bb0378681d8f0e3cf62a7f207014263ec7be3",
},
{
vcs: "git",
@ -201,6 +217,8 @@ var codeRepoTests = []codeRepoTest{
"pkg/p.go",
"LICENSE",
},
zipSum: "h1:iMsJ/9uQsk6MnZNnJK311f11QiSlmN92Q2aSjCywuJY=",
zipFileHash: "95801bfa69c5197ae809af512946d22f22850068527cd78100ae3f176bc8043b",
},
{
vcs: "git",
@ -217,16 +235,20 @@ var codeRepoTests = []codeRepoTest{
"go.mod",
"pkg/p.go",
},
zipSum: "h1:M69k7q+8bQ+QUpHov45Z/NoR8rj3DsQJUnXLWvf01+Q=",
zipFileHash: "58af45fb248d320ea471f568e006379e2b8d71d6d1663f9b19b2e00fd9ac9265",
},
{
vcs: "git",
path: "github.com/rsc/vgotest1/v2",
rev: "v2.0.1",
version: "v2.0.1",
name: "ea65f87c8f52c15ea68f3bdd9925ef17e20d91e9",
short: "ea65f87c8f52",
time: time.Date(2018, 2, 19, 23, 14, 23, 0, time.UTC),
gomod: "module \"github.com/rsc/vgotest1/v2\" // root go.mod\n",
vcs: "git",
path: "github.com/rsc/vgotest1/v2",
rev: "v2.0.1",
version: "v2.0.1",
name: "ea65f87c8f52c15ea68f3bdd9925ef17e20d91e9",
short: "ea65f87c8f52",
time: time.Date(2018, 2, 19, 23, 14, 23, 0, time.UTC),
gomod: "module \"github.com/rsc/vgotest1/v2\" // root go.mod\n",
zipSum: "h1:QmgYy/zt+uoWhDpcsgrSVzYFvKtBEjl5zT/FRz9GTzA=",
zipFileHash: "1aedf1546d322a0121879ddfd6d0e8bfbd916d2cafbeb538ddb440e04b04b9ef",
},
{
vcs: "git",
@ -249,25 +271,29 @@ var codeRepoTests = []codeRepoTest{
err: "github.com/rsc/vgotest1/go.mod and .../v2/go.mod both have .../v2 module paths at revision v2.0.4",
},
{
vcs: "git",
path: "github.com/rsc/vgotest1/v2",
rev: "v2.0.5",
version: "v2.0.5",
name: "2f615117ce481c8efef46e0cc0b4b4dccfac8fea",
short: "2f615117ce48",
time: time.Date(2018, 2, 20, 0, 3, 59, 0, time.UTC),
gomod: "module \"github.com/rsc/vgotest1/v2\" // v2/go.mod\n",
vcs: "git",
path: "github.com/rsc/vgotest1/v2",
rev: "v2.0.5",
version: "v2.0.5",
name: "2f615117ce481c8efef46e0cc0b4b4dccfac8fea",
short: "2f615117ce48",
time: time.Date(2018, 2, 20, 0, 3, 59, 0, time.UTC),
gomod: "module \"github.com/rsc/vgotest1/v2\" // v2/go.mod\n",
zipSum: "h1:RIEb9q1SUSEQOzMn0zfl/LQxGFWlhWEAdeEguf1MLGU=",
zipFileHash: "7d92c2c328c5e9b0694101353705d5843746ec1d93a1e986d0da54c8a14dfe6d",
},
{
// redirect to github
vcs: "git",
path: "rsc.io/quote",
rev: "v1.0.0",
version: "v1.0.0",
name: "f488df80bcdbd3e5bafdc24ad7d1e79e83edd7e6",
short: "f488df80bcdb",
time: time.Date(2018, 2, 14, 0, 45, 20, 0, time.UTC),
gomod: "module \"rsc.io/quote\"\n",
vcs: "git",
path: "rsc.io/quote",
rev: "v1.0.0",
version: "v1.0.0",
name: "f488df80bcdbd3e5bafdc24ad7d1e79e83edd7e6",
short: "f488df80bcdb",
time: time.Date(2018, 2, 14, 0, 45, 20, 0, time.UTC),
gomod: "module \"rsc.io/quote\"\n",
zipSum: "h1:haUSojyo3j2M9g7CEUFG8Na09dtn7QKxvPGaPVQdGwM=",
zipFileHash: "5c08ba2c09a364f93704aaa780e7504346102c6ef4fe1333a11f09904a732078",
},
{
// redirect to static hosting proxy
@ -281,22 +307,26 @@ var codeRepoTests = []codeRepoTest{
},
{
// redirect to googlesource
vcs: "git",
path: "golang.org/x/text",
rev: "4e4a3210bb",
version: "v0.3.1-0.20180208041248-4e4a3210bb54",
name: "4e4a3210bb54bb31f6ab2cdca2edcc0b50c420c1",
short: "4e4a3210bb54",
time: time.Date(2018, 2, 8, 4, 12, 48, 0, time.UTC),
vcs: "git",
path: "golang.org/x/text",
rev: "4e4a3210bb",
version: "v0.3.1-0.20180208041248-4e4a3210bb54",
name: "4e4a3210bb54bb31f6ab2cdca2edcc0b50c420c1",
short: "4e4a3210bb54",
time: time.Date(2018, 2, 8, 4, 12, 48, 0, time.UTC),
zipSum: "h1:Yxu6pHX9X2RECiuw/Q5/4uvajuaowck8zOFKXgbfNBk=",
zipFileHash: "ac2c165a5c10aa5a7545dea60a08e019270b982fa6c8bdcb5943931de64922fe",
},
{
vcs: "git",
path: "github.com/pkg/errors",
rev: "v0.8.0",
version: "v0.8.0",
name: "645ef00459ed84a119197bfb8d8205042c6df63d",
short: "645ef00459ed",
time: time.Date(2016, 9, 29, 1, 48, 1, 0, time.UTC),
vcs: "git",
path: "github.com/pkg/errors",
rev: "v0.8.0",
version: "v0.8.0",
name: "645ef00459ed84a119197bfb8d8205042c6df63d",
short: "645ef00459ed",
time: time.Date(2016, 9, 29, 1, 48, 1, 0, time.UTC),
zipSum: "h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=",
zipFileHash: "e4fa69ba057356614edbc1da881a7d3ebb688505be49f65965686bcb859e2fae",
},
{
// package in subdirectory - custom domain
@ -304,7 +334,7 @@ var codeRepoTests = []codeRepoTest{
// but gopkg.in is special.
vcs: "git",
path: "gopkg.in/yaml.v2/abc",
lookerr: "invalid module path \"gopkg.in/yaml.v2/abc\"",
lookErr: "invalid module path \"gopkg.in/yaml.v2/abc\"",
},
{
// package in subdirectory - github
@ -315,54 +345,52 @@ var codeRepoTests = []codeRepoTest{
err: "missing github.com/rsc/quote/buggy/go.mod at revision c4d4236f9242",
},
{
vcs: "git",
path: "gopkg.in/yaml.v2",
rev: "d670f940",
version: "v2.0.0",
name: "d670f9405373e636a5a2765eea47fac0c9bc91a4",
short: "d670f9405373",
time: time.Date(2018, 1, 9, 11, 43, 31, 0, time.UTC),
gomod: "module gopkg.in/yaml.v2\n",
vcs: "git",
path: "gopkg.in/yaml.v2",
rev: "d670f940",
version: "v2.0.0",
name: "d670f9405373e636a5a2765eea47fac0c9bc91a4",
short: "d670f9405373",
time: time.Date(2018, 1, 9, 11, 43, 31, 0, time.UTC),
gomod: "module gopkg.in/yaml.v2\n",
zipSum: "h1:uUkhRGrsEyx/laRdeS6YIQKIys8pg+lRSRdVMTYjivs=",
zipFileHash: "7b0a141b1b0b49772ab4eecfd11dfd6609a94a5e868cab04a3abb1861ffaa877",
},
{
vcs: "git",
path: "gopkg.in/check.v1",
rev: "20d25e280405",
version: "v1.0.0-20161208181325-20d25e280405",
name: "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec",
short: "20d25e280405",
time: time.Date(2016, 12, 8, 18, 13, 25, 0, time.UTC),
gomod: "module gopkg.in/check.v1\n",
vcs: "git",
path: "gopkg.in/check.v1",
rev: "20d25e280405",
version: "v1.0.0-20161208181325-20d25e280405",
name: "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec",
short: "20d25e280405",
time: time.Date(2016, 12, 8, 18, 13, 25, 0, time.UTC),
gomod: "module gopkg.in/check.v1\n",
zipSum: "h1:829vOVxxusYHC+IqBtkX5mbKtsY9fheQiQn0MZRVLfQ=",
zipFileHash: "9e7cb3f4f1e66d722306442b0dbe1f6f43d74d1736d54c510537bdfb1d6f432f",
},
{
vcs: "git",
path: "gopkg.in/yaml.v2",
rev: "v2",
version: "v2.2.3-0.20190319135612-7b8349ac747c",
name: "7b8349ac747c6a24702b762d2c4fd9266cf4f1d6",
short: "7b8349ac747c",
time: time.Date(2019, 03, 19, 13, 56, 12, 0, time.UTC),
gomod: "module \"gopkg.in/yaml.v2\"\n\nrequire (\n\t\"gopkg.in/check.v1\" v0.0.0-20161208181325-20d25e280405\n)\n",
vcs: "git",
path: "vcs-test.golang.org/go/mod/gitrepo1",
rev: "master",
version: "v1.2.4-annotated",
name: "ede458df7cd0fdca520df19a33158086a8a68e81",
short: "ede458df7cd0",
time: time.Date(2018, 4, 17, 19, 43, 22, 0, time.UTC),
gomod: "module vcs-test.golang.org/go/mod/gitrepo1\n",
zipSum: "h1:YJYZRsM9BHFTlVr8YADjT0cJH8uFIDtoc5NLiVqZEx8=",
zipFileHash: "c15e49d58b7a4c37966cbe5bc01a0330cd5f2927e990e1839bda1d407766d9c5",
},
{
vcs: "git",
path: "vcs-test.golang.org/go/mod/gitrepo1",
rev: "master",
version: "v1.2.4-annotated",
name: "ede458df7cd0fdca520df19a33158086a8a68e81",
short: "ede458df7cd0",
time: time.Date(2018, 4, 17, 19, 43, 22, 0, time.UTC),
gomod: "module vcs-test.golang.org/go/mod/gitrepo1\n",
},
{
vcs: "git",
path: "gopkg.in/natefinch/lumberjack.v2",
rev: "latest",
version: "v2.0.0-20170531160350-a96e63847dc3",
name: "a96e63847dc3c67d17befa69c303767e2f84e54f",
short: "a96e63847dc3",
time: time.Date(2017, 5, 31, 16, 3, 50, 0, time.UTC),
gomod: "module gopkg.in/natefinch/lumberjack.v2\n",
vcs: "git",
path: "gopkg.in/natefinch/lumberjack.v2",
rev: "latest",
version: "v2.0.0-20170531160350-a96e63847dc3",
name: "a96e63847dc3c67d17befa69c303767e2f84e54f",
short: "a96e63847dc3",
time: time.Date(2017, 5, 31, 16, 3, 50, 0, time.UTC),
gomod: "module gopkg.in/natefinch/lumberjack.v2\n",
zipSum: "h1:AFxeG48hTWHhDTQDk/m2gorfVHUEa9vo3tp3D7TzwjI=",
zipFileHash: "b5de0da7bbbec76709eef1ac71b6c9ff423b9fbf3bb97b56743450d4937b06d5",
},
{
vcs: "git",
@ -381,14 +409,16 @@ var codeRepoTests = []codeRepoTest{
gomod: "module gopkg.in/natefinch/lumberjack.v2\n",
},
{
vcs: "git",
path: "vcs-test.golang.org/go/v2module/v2",
rev: "v2.0.0",
version: "v2.0.0",
name: "203b91c896acd173aa719e4cdcb7d463c4b090fa",
short: "203b91c896ac",
time: time.Date(2019, 4, 3, 15, 52, 15, 0, time.UTC),
gomod: "module vcs-test.golang.org/go/v2module/v2\n\ngo 1.12\n",
vcs: "git",
path: "vcs-test.golang.org/go/v2module/v2",
rev: "v2.0.0",
version: "v2.0.0",
name: "203b91c896acd173aa719e4cdcb7d463c4b090fa",
short: "203b91c896ac",
time: time.Date(2019, 4, 3, 15, 52, 15, 0, time.UTC),
gomod: "module vcs-test.golang.org/go/v2module/v2\n\ngo 1.12\n",
zipSum: "h1:JItBZ+gwA5WvtZEGEbuDL4lUttGtLrs53lmdurq3bOg=",
zipFileHash: "9ea9ae1673cffcc44b7fdd3cc89953d68c102449b46c982dbf085e4f2e394da5",
},
}
@ -411,21 +441,23 @@ func TestCodeRepo(t *testing.T) {
}
repo, err := Lookup("direct", tt.path)
if tt.lookerr != "" {
if err != nil && err.Error() == tt.lookerr {
if tt.lookErr != "" {
if err != nil && err.Error() == tt.lookErr {
return
}
t.Errorf("Lookup(%q): %v, want error %q", tt.path, err, tt.lookerr)
t.Errorf("Lookup(%q): %v, want error %q", tt.path, err, tt.lookErr)
}
if err != nil {
t.Fatalf("Lookup(%q): %v", tt.path, err)
}
if tt.mpath == "" {
tt.mpath = tt.path
}
if mpath := repo.ModulePath(); mpath != tt.mpath {
t.Errorf("repo.ModulePath() = %q, want %q", mpath, tt.mpath)
}
info, err := repo.Stat(tt.rev)
if err != nil {
if tt.err != "" {
@ -451,56 +483,86 @@ func TestCodeRepo(t *testing.T) {
if !info.Time.Equal(tt.time) {
t.Errorf("info.Time = %v, want %v", info.Time, tt.time)
}
if tt.gomod != "" || tt.gomoderr != "" {
if tt.gomod != "" || tt.gomodErr != "" {
data, err := repo.GoMod(tt.version)
if err != nil && tt.gomoderr == "" {
if err != nil && tt.gomodErr == "" {
t.Errorf("repo.GoMod(%q): %v", tt.version, err)
} else if err != nil && tt.gomoderr != "" {
if err.Error() != tt.gomoderr {
t.Errorf("repo.GoMod(%q): %v, want %q", tt.version, err, tt.gomoderr)
} else if err != nil && tt.gomodErr != "" {
if err.Error() != tt.gomodErr {
t.Errorf("repo.GoMod(%q): %v, want %q", tt.version, err, tt.gomodErr)
}
} else if tt.gomoderr != "" {
t.Errorf("repo.GoMod(%q) = %q, want error %q", tt.version, data, tt.gomoderr)
} else if tt.gomodErr != "" {
t.Errorf("repo.GoMod(%q) = %q, want error %q", tt.version, data, tt.gomodErr)
} else if string(data) != tt.gomod {
t.Errorf("repo.GoMod(%q) = %q, want %q", tt.version, data, tt.gomod)
}
}
if tt.zip != nil || tt.ziperr != "" {
needHash := !testing.Short() && (tt.zipFileHash != "" || tt.zipSum != "")
if tt.zip != nil || tt.zipErr != "" || needHash {
f, err := ioutil.TempFile(tmpdir, tt.version+".zip.")
if err != nil {
t.Fatalf("ioutil.TempFile: %v", err)
}
zipfile := f.Name()
err = repo.Zip(f, tt.version)
defer func() {
f.Close()
os.Remove(zipfile)
}()
var w io.Writer
var h hash.Hash
if needHash {
h = sha256.New()
w = io.MultiWriter(f, h)
} else {
w = f
}
err = repo.Zip(w, tt.version)
f.Close()
if err != nil {
if tt.ziperr != "" {
if err.Error() == tt.ziperr {
if tt.zipErr != "" {
if err.Error() == tt.zipErr {
return
}
t.Fatalf("repo.Zip(%q): %v, want error %q", tt.version, err, tt.ziperr)
t.Fatalf("repo.Zip(%q): %v, want error %q", tt.version, err, tt.zipErr)
}
t.Fatalf("repo.Zip(%q): %v", tt.version, err)
}
if tt.ziperr != "" {
t.Errorf("repo.Zip(%q): success, want error %q", tt.version, tt.ziperr)
if tt.zipErr != "" {
t.Errorf("repo.Zip(%q): success, want error %q", tt.version, tt.zipErr)
}
prefix := tt.path + "@" + tt.version + "/"
z, err := zip.OpenReader(zipfile)
if err != nil {
t.Fatalf("open zip %s: %v", zipfile, err)
}
var names []string
for _, file := range z.File {
if !strings.HasPrefix(file.Name, prefix) {
t.Errorf("zip entry %v does not start with prefix %v", file.Name, prefix)
continue
if tt.zip != nil {
prefix := tt.path + "@" + tt.version + "/"
z, err := zip.OpenReader(zipfile)
if err != nil {
t.Fatalf("open zip %s: %v", zipfile, err)
}
var names []string
for _, file := range z.File {
if !strings.HasPrefix(file.Name, prefix) {
t.Errorf("zip entry %v does not start with prefix %v", file.Name, prefix)
continue
}
names = append(names, file.Name[len(prefix):])
}
z.Close()
if !reflect.DeepEqual(names, tt.zip) {
t.Fatalf("zip = %v\nwant %v\n", names, tt.zip)
}
names = append(names, file.Name[len(prefix):])
}
z.Close()
if !reflect.DeepEqual(names, tt.zip) {
t.Fatalf("zip = %v\nwant %v\n", names, tt.zip)
if needHash {
sum, err := dirhash.HashZip(zipfile, dirhash.Hash1)
if err != nil {
t.Errorf("repo.Zip(%q): %v", tt.version, err)
} else if sum != tt.zipSum {
t.Errorf("repo.Zip(%q): got file with sum %q, want %q", tt.version, sum, tt.zipSum)
} else if zipFileHash := hex.EncodeToString(h.Sum(nil)); zipFileHash != tt.zipFileHash {
t.Errorf("repo.Zip(%q): got file with hash %q, want %q (but content has correct sum)", tt.version, zipFileHash, tt.zipFileHash)
}
}
}
}
@ -508,26 +570,26 @@ func TestCodeRepo(t *testing.T) {
t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.rev, f(tt))
if strings.HasPrefix(tt.path, vgotest1git) {
for vcs, alt := range altVgotests {
// Note: Communicating with f through tt; should be cleaned up.
old := tt
tt.vcs = vcs
tt.path = alt + strings.TrimPrefix(tt.path, vgotest1git)
if strings.HasPrefix(tt.mpath, vgotest1git) {
tt.mpath = alt + strings.TrimPrefix(tt.mpath, vgotest1git)
altTest := tt
altTest.vcs = vcs
altTest.path = alt + strings.TrimPrefix(altTest.path, vgotest1git)
if strings.HasPrefix(altTest.mpath, vgotest1git) {
altTest.mpath = alt + strings.TrimPrefix(altTest.mpath, vgotest1git)
}
var m map[string]string
if alt == vgotest1hg {
m = hgmap
}
tt.version = remap(tt.version, m)
tt.name = remap(tt.name, m)
tt.short = remap(tt.short, m)
tt.rev = remap(tt.rev, m)
tt.err = remap(tt.err, m)
tt.gomoderr = remap(tt.gomoderr, m)
tt.ziperr = remap(tt.ziperr, m)
t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.rev, f(tt))
tt = old
altTest.version = remap(altTest.version, m)
altTest.name = remap(altTest.name, m)
altTest.short = remap(altTest.short, m)
altTest.rev = remap(altTest.rev, m)
altTest.err = remap(altTest.err, m)
altTest.gomodErr = remap(altTest.gomodErr, m)
altTest.zipErr = remap(altTest.zipErr, m)
altTest.zipSum = ""
altTest.zipFileHash = ""
t.Run(strings.ReplaceAll(altTest.path, "/", "_")+"/"+altTest.rev, f(altTest))
}
}
}
@ -575,7 +637,7 @@ var codeRepoVersionsTests = []struct {
{
vcs: "git",
path: "github.com/rsc/vgotest1",
versions: []string{"v0.0.0", "v0.0.1", "v1.0.0", "v1.0.1", "v1.0.2", "v1.0.3", "v1.1.0", "v2.0.0+incompatible"},
versions: []string{"v0.0.0", "v0.0.1", "v1.0.0", "v1.0.1", "v1.0.2", "v1.0.3", "v1.1.0"},
},
{
vcs: "git",

218
libgo/go/cmd/go/internal/modfetch/fetch.go
View file

@ -7,6 +7,7 @@ package modfetch
import (
"archive/zip"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
@ -18,10 +19,13 @@ import (
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/dirhash"
"cmd/go/internal/module"
"cmd/go/internal/lockedfile"
"cmd/go/internal/par"
"cmd/go/internal/renameio"
"golang.org/x/mod/module"
"golang.org/x/mod/sumdb/dirhash"
modzip "golang.org/x/mod/zip"
)
var downloadCache par.Cache
@ -69,10 +73,6 @@ func download(mod module.Version, dir string) (err error) {
return err
}
if cfg.CmdName != "mod download" {
fmt.Fprintf(os.Stderr, "go: extracting %s %s\n", mod.Path, mod.Version)
}
unlock, err := lockVersion(mod)
if err != nil {
return err
@ -114,8 +114,7 @@ func download(mod module.Version, dir string) (err error) {
}
}()
modpath := mod.Path + "@" + mod.Version
if err := Unzip(tmpDir, zipfile, modpath, 0); err != nil {
if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil {
fmt.Fprintf(os.Stderr, "-> %s\n", err)
return err
}
@ -124,9 +123,11 @@ func download(mod module.Version, dir string) (err error) {
return err
}
// Make dir read-only only *after* renaming it.
// os.Rename was observed to fail for read-only directories on macOS.
makeDirsReadOnly(dir)
if !cfg.ModCacheRW {
// Make dir read-only only *after* renaming it.
// os.Rename was observed to fail for read-only directories on macOS.
makeDirsReadOnly(dir)
}
return nil
}
@ -249,7 +250,9 @@ func downloadZip(mod module.Version, zipfile string) (err error) {
if err != nil {
return err
}
checkModSum(mod, hash)
if err := checkModSum(mod, hash); err != nil {
return err
}
if err := renameio.WriteFile(zipfile+"hash", []byte(hash), 0666); err != nil {
return err
@ -263,6 +266,45 @@ func downloadZip(mod module.Version, zipfile string) (err error) {
return nil
}
// makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
// and its transitive contents.
func makeDirsReadOnly(dir string) {
type pathMode struct {
path string
mode os.FileMode
}
var dirs []pathMode // in lexical order
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err == nil && info.Mode()&0222 != 0 {
if info.IsDir() {
dirs = append(dirs, pathMode{path, info.Mode()})
}
}
return nil
})
// Run over list backward to chmod children before parents.
for i := len(dirs) - 1; i >= 0; i-- {
os.Chmod(dirs[i].path, dirs[i].mode&^0222)
}
}
// RemoveAll removes a directory written by Download or Unzip, first applying
// any permission changes needed to do so.
func RemoveAll(dir string) error {
// Module cache has 0555 directories; make them writable in order to remove content.
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil // ignore errors walking in file system
}
if info.IsDir() {
os.Chmod(path, 0777)
}
return nil
})
return os.RemoveAll(dir)
}
var GoSumFile string // path to go.sum; set by package modload
type modSum struct {
@ -281,21 +323,22 @@ var goSum struct {
}
// initGoSum initializes the go.sum data.
// It reports whether use of go.sum is now enabled.
// The boolean it returns reports whether the
// use of go.sum is now enabled.
// The goSum lock must be held.
func initGoSum() bool {
func initGoSum() (bool, error) {
if GoSumFile == "" {
return false
return false, nil
}
if goSum.m != nil {
return true
return true, nil
}
goSum.m = make(map[module.Version][]string)
goSum.checked = make(map[modSum]bool)
data, err := renameio.ReadFile(GoSumFile)
data, err := lockedfile.Read(GoSumFile)
if err != nil && !os.IsNotExist(err) {
base.Fatalf("go: %v", err)
return false, err
}
goSum.enabled = true
readGoSum(goSum.m, GoSumFile, data)
@ -313,7 +356,7 @@ func initGoSum() bool {
}
goSum.modverify = alt
}
return true
return true, nil
}
// emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
@ -323,7 +366,7 @@ const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
// readGoSum parses data, which is the content of file,
// and adds it to goSum.m. The goSum lock must be held.
func readGoSum(dst map[module.Version][]string, file string, data []byte) {
func readGoSum(dst map[module.Version][]string, file string, data []byte) error {
lineno := 0
for len(data) > 0 {
var line []byte
@ -340,7 +383,7 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) {
continue
}
if len(f) != 3 {
base.Fatalf("go: malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
return fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
}
if f[2] == emptyGoModHash {
// Old bug; drop it.
@ -349,6 +392,7 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) {
mod := module.Version{Path: f[0], Version: f[1]}
dst[mod] = append(dst[mod], f[2])
}
return nil
}
// checkMod checks the given module's checksum.
@ -361,22 +405,24 @@ func checkMod(mod module.Version) {
// Do the file I/O before acquiring the go.sum lock.
ziphash, err := CachePath(mod, "ziphash")
if err != nil {
base.Fatalf("verifying %s@%s: %v", mod.Path, mod.Version, err)
base.Fatalf("verifying %v", module.VersionError(mod, err))
}
data, err := renameio.ReadFile(ziphash)
if err != nil {
if os.IsNotExist(err) {
if errors.Is(err, os.ErrNotExist) {
// This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes.
return
}
base.Fatalf("verifying %s@%s: %v", mod.Path, mod.Version, err)
base.Fatalf("verifying %v", module.VersionError(mod, err))
}
h := strings.TrimSpace(string(data))
if !strings.HasPrefix(h, "h1:") {
base.Fatalf("verifying %s@%s: unexpected ziphash: %q", mod.Path, mod.Version, h)
base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
}
checkModSum(mod, h)
if err := checkModSum(mod, h); err != nil {
base.Fatalf("%s", err)
}
}
// goModSum returns the checksum for the go.mod contents.
@ -388,17 +434,17 @@ func goModSum(data []byte) (string, error) {
// checkGoMod checks the given module's go.mod checksum;
// data is the go.mod content.
func checkGoMod(path, version string, data []byte) {
func checkGoMod(path, version string, data []byte) error {
h, err := goModSum(data)
if err != nil {
base.Fatalf("verifying %s %s go.mod: %v", path, version, err)
return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
}
checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
return checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
}
// checkModSum checks that the recorded checksum for mod is h.
func checkModSum(mod module.Version, h string) {
func checkModSum(mod module.Version, h string) error {
// We lock goSum when manipulating it,
// but we arrange to release the lock when calling checkSumDB,
// so that parallel calls to checkModHash can execute parallel calls
@ -406,19 +452,24 @@ func checkModSum(mod module.Version, h string) {
// Check whether mod+h is listed in go.sum already. If so, we're done.
goSum.mu.Lock()
inited := initGoSum()
inited, err := initGoSum()
if err != nil {
return err
}
done := inited && haveModSumLocked(mod, h)
goSum.mu.Unlock()
if done {
return
return nil
}
// Not listed, so we want to add them.
// Consult checksum database if appropriate.
if useSumDB(mod) {
// Calls base.Fatalf if mismatch detected.
checkSumDB(mod, h)
if err := checkSumDB(mod, h); err != nil {
return err
}
}
// Add mod+h to go.sum, if it hasn't appeared already.
@ -427,6 +478,7 @@ func checkModSum(mod module.Version, h string) {
addModSumLocked(mod, h)
goSum.mu.Unlock()
}
return nil
}
// haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
@ -460,22 +512,23 @@ func addModSumLocked(mod module.Version, h string) {
// checkSumDB checks the mod, h pair against the Go checksum database.
// It calls base.Fatalf if the hash is to be rejected.
func checkSumDB(mod module.Version, h string) {
func checkSumDB(mod module.Version, h string) error {
db, lines, err := lookupSumDB(mod)
if err != nil {
base.Fatalf("verifying %s@%s: %v", mod.Path, mod.Version, err)
return module.VersionError(mod, fmt.Errorf("verifying module: %v", err))
}
have := mod.Path + " " + mod.Version + " " + h
prefix := mod.Path + " " + mod.Version + " h1:"
for _, line := range lines {
if line == have {
return
return nil
}
if strings.HasPrefix(line, prefix) {
base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, mod.Path, mod.Version, h, db, line[len(prefix)-len("h1:"):])
return module.VersionError(mod, fmt.Errorf("verifying module: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, h, db, line[len(prefix)-len("h1:"):]))
}
}
return nil
}
// Sum returns the checksum for the downloaded copy of the given module,
@ -516,60 +569,45 @@ func WriteGoSum() {
base.Fatalf("go: updates to go.sum needed, disabled by -mod=readonly")
}
// We want to avoid races between creating the lockfile and deleting it, but
// we also don't want to leave a permanent lockfile in the user's repository.
//
// On top of that, if we crash while writing go.sum, we don't want to lose the
// sums that were already present in the file, so it's important that we write
// the file by renaming rather than truncating — which means that we can't
// lock the go.sum file itself.
//
// Instead, we'll lock a distinguished file in the cache directory: that will
// only race if the user runs `go clean -modcache` concurrently with a command
// that updates go.sum, and that's already racy to begin with.
//
// We'll end up slightly over-synchronizing go.sum writes if the user runs a
// bunch of go commands that update sums in separate modules simultaneously,
// but that's unlikely to matter in practice.
unlock := SideLock()
defer unlock()
if !goSum.overwrite {
// Re-read the go.sum file to incorporate any sums added by other processes
// in the meantime.
data, err := renameio.ReadFile(GoSumFile)
if err != nil && !os.IsNotExist(err) {
base.Fatalf("go: re-reading go.sum: %v", err)
}
// Add only the sums that we actually checked: the user may have edited or
// truncated the file to remove erroneous hashes, and we shouldn't restore
// them without good reason.
goSum.m = make(map[module.Version][]string, len(goSum.m))
readGoSum(goSum.m, GoSumFile, data)
for ms := range goSum.checked {
addModSumLocked(ms.mod, ms.sum)
goSum.dirty = true
}
// Make a best-effort attempt to acquire the side lock, only to exclude
// previous versions of the 'go' command from making simultaneous edits.
if unlock, err := SideLock(); err == nil {
defer unlock()
}
var mods []module.Version
for m := range goSum.m {
mods = append(mods, m)
}
module.Sort(mods)
var buf bytes.Buffer
for _, m := range mods {
list := goSum.m[m]
sort.Strings(list)
for _, h := range list {
fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
err := lockedfile.Transform(GoSumFile, func(data []byte) ([]byte, error) {
if !goSum.overwrite {
// Incorporate any sums added by other processes in the meantime.
// Add only the sums that we actually checked: the user may have edited or
// truncated the file to remove erroneous hashes, and we shouldn't restore
// them without good reason.
goSum.m = make(map[module.Version][]string, len(goSum.m))
readGoSum(goSum.m, GoSumFile, data)
for ms := range goSum.checked {
addModSumLocked(ms.mod, ms.sum)
goSum.dirty = true
}
}
}
if err := renameio.WriteFile(GoSumFile, buf.Bytes(), 0666); err != nil {
base.Fatalf("go: writing go.sum: %v", err)
var mods []module.Version
for m := range goSum.m {
mods = append(mods, m)
}
module.Sort(mods)
var buf bytes.Buffer
for _, m := range mods {
list := goSum.m[m]
sort.Strings(list)
for _, h := range list {
fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
}
}
return buf.Bytes(), nil
})
if err != nil {
base.Fatalf("go: updating go.sum: %v", err)
}
goSum.checked = make(map[modSum]bool)
@ -585,7 +623,11 @@ func WriteGoSum() {
func TrimGoSum(keep map[module.Version]bool) {
goSum.mu.Lock()
defer goSum.mu.Unlock()
if !initGoSum() {
inited, err := initGoSum()
if err != nil {
base.Fatalf("%s", err)
}
if !inited {
return
}

16
libgo/go/cmd/go/internal/modfetch/insecure.go Normal file
View file

@ -0,0 +1,16 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package modfetch
import (
"cmd/go/internal/cfg"
"cmd/go/internal/get"
"cmd/go/internal/str"
)
// allowInsecure reports whether we are allowed to fetch this path in an insecure manner.
func allowInsecure(path string) bool {
return get.Insecure || str.GlobsMatchPath(cfg.GOINSECURE, path)
}

49
libgo/go/cmd/go/internal/modfetch/proxy.go
View file

@ -22,9 +22,10 @@ import (
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/module"
"cmd/go/internal/semver"
"cmd/go/internal/web"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
)
var HelpGoproxy = &base.Command{
@ -38,8 +39,8 @@ can be a module proxy.
The GET requests sent to a Go module proxy are:
GET $GOPROXY/<module>/@v/list returns a list of all known versions of the
given module, one per line.
GET $GOPROXY/<module>/@v/list returns a list of known versions of the given
module, one per line.
GET $GOPROXY/<module>/@v/<version>.info returns JSON-formatted metadata
about that version of the given module.
@ -50,6 +51,21 @@ for that version of the given module.
GET $GOPROXY/<module>/@v/<version>.zip returns the zip archive
for that version of the given module.
GET $GOPROXY/<module>/@latest returns JSON-formatted metadata about the
latest known version of the given module in the same format as
<module>/@v/<version>.info. The latest version should be the version of
the module the go command may use if <module>/@v/list is empty or no
listed version is suitable. <module>/@latest is optional and may not
be implemented by a module proxy.
When resolving the latest version of a module, the go command will request
<module>/@v/list, then, if no suitable versions are found, <module>/@latest.
The go command prefers, in order: the semantically highest release version,
the semantically highest pre-release version, and the chronologically
most recent pseudo-version. In Go 1.12 and earlier, the go command considered
pseudo-versions in <module>/@v/list to be pre-release versions, but this is
no longer true since Go 1.13.
To avoid problems when serving from case-sensitive file systems,
the <module> and <version> elements are case-encoded, replacing every
uppercase letter with an exclamation mark followed by the corresponding
@ -150,13 +166,28 @@ func TryProxies(f func(proxy string) error) error {
return f("off")
}
var lastAttemptErr error
for _, proxy := range proxies {
err = f(proxy)
if !errors.Is(err, os.ErrNotExist) {
lastAttemptErr = err
break
}
// The error indicates that the module does not exist.
// In general we prefer to report the last such error,
// because it indicates the error that occurs after all other
// options have been exhausted.
//
// However, for modules in the NOPROXY list, the most useful error occurs
// first (with proxy set to "noproxy"), and the subsequent errors are all
// errNoProxy (which is not particularly helpful). Do not overwrite a more
// useful error with errNoproxy.
if lastAttemptErr == nil || !errors.Is(err, errNoproxy) {
lastAttemptErr = err
}
}
return err
return lastAttemptErr
}
type proxyRepo struct {
@ -182,7 +213,7 @@ func newProxyRepo(baseURL, path string) (Repo, error) {
return nil, fmt.Errorf("invalid proxy URL scheme (must be https, http, file): %s", web.Redacted(base))
}
enc, err := module.EncodePath(path)
enc, err := module.EscapePath(path)
if err != nil {
return nil, err
}
@ -321,7 +352,7 @@ func (p *proxyRepo) latest() (*RevInfo, error) {
}
func (p *proxyRepo) Stat(rev string) (*RevInfo, error) {
encRev, err := module.EncodeVersion(rev)
encRev, err := module.EscapeVersion(rev)
if err != nil {
return nil, p.versionError(rev, err)
}
@ -362,7 +393,7 @@ func (p *proxyRepo) GoMod(version string) ([]byte, error) {
return nil, p.versionError(version, fmt.Errorf("internal error: version passed to GoMod is not canonical"))
}
encVer, err := module.EncodeVersion(version)
encVer, err := module.EscapeVersion(version)
if err != nil {
return nil, p.versionError(version, err)
}
@ -378,7 +409,7 @@ func (p *proxyRepo) Zip(dst io.Writer, version string) error {
return p.versionError(version, fmt.Errorf("internal error: version passed to Zip is not canonical"))
}
encVer, err := module.EncodeVersion(version)
encVer, err := module.EscapeVersion(version)
if err != nil {
return p.versionError(version, err)
}

5
libgo/go/cmd/go/internal/modfetch/pseudo.go
View file

@ -40,9 +40,10 @@ import (
"strings"
"time"
"cmd/go/internal/module"
"cmd/go/internal/semver"
"internal/lazyregexp"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
)
var pseudoVersionRE = lazyregexp.New(`^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)\d{14}-[A-Za-z0-9]+(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`)

50
libgo/go/cmd/go/internal/modfetch/repo.go
View file

@ -5,7 +5,6 @@
package modfetch
import (
"errors"
"fmt"
"io"
"os"
@ -17,9 +16,10 @@ import (
"cmd/go/internal/get"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/par"
"cmd/go/internal/semver"
"cmd/go/internal/str"
web "cmd/go/internal/web"
"golang.org/x/mod/semver"
)
const traceRepo = false // trace all repo actions, for debugging
@ -34,7 +34,7 @@ type Repo interface {
// Pseudo-versions are not included.
// Versions should be returned sorted in semver order
// (implementations can use SortVersions).
Versions(prefix string) (tags []string, err error)
Versions(prefix string) ([]string, error)
// Stat returns information about the revision rev.
// A revision can be any identifier known to the underlying service:
@ -55,7 +55,7 @@ type Repo interface {
// A Rev describes a single revision in a module repository.
type RevInfo struct {
Version string // version string
Version string // suggested version string for this revision
Time time.Time // commit time
// These fields are used for Stat of arbitrary rev,
@ -214,7 +214,7 @@ func Lookup(proxy, path string) (Repo, error) {
// lookup returns the module with the given module path.
func lookup(proxy, path string) (r Repo, err error) {
if cfg.BuildMod == "vendor" {
return nil, errModVendor
return nil, errLookupDisabled
}
if str.GlobsMatchPath(cfg.GONOPROXY, path) {
@ -238,22 +238,33 @@ func lookup(proxy, path string) (r Repo, err error) {
}
}
type lookupDisabledError struct{}
func (lookupDisabledError) Error() string {
if cfg.BuildModReason == "" {
return fmt.Sprintf("module lookup disabled by -mod=%s", cfg.BuildMod)
}
return fmt.Sprintf("module lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason)
}
var errLookupDisabled error = lookupDisabledError{}
var (
errModVendor = errors.New("module lookup disabled by -mod=vendor")
errProxyOff = notExistError("module lookup disabled by GOPROXY=off")
errNoproxy error = notExistError("disabled by GOPRIVATE/GONOPROXY")
errUseProxy error = notExistError("path does not match GOPRIVATE/GONOPROXY")
errProxyOff = notExistErrorf("module lookup disabled by GOPROXY=off")
errNoproxy error = notExistErrorf("disabled by GOPRIVATE/GONOPROXY")
errUseProxy error = notExistErrorf("path does not match GOPRIVATE/GONOPROXY")
)
func lookupDirect(path string) (Repo, error) {
security := web.SecureOnly
if get.Insecure {
if allowInsecure(path) {
security = web.Insecure
}
rr, err := get.RepoRootForImportPath(path, get.PreferMod, security)
if err != nil {
// We don't know where to find code for a module with this path.
return nil, notExistError(err.Error())
return nil, notExistError{err: err}
}
if rr.VCS == "mod" {
@ -292,7 +303,7 @@ func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) {
// version control system, we ignore meta tags about modules
// and use only direct source control entries (get.IgnoreMod).
security := web.SecureOnly
if get.Insecure {
if allowInsecure(path) {
security = web.Insecure
}
rr, err := get.RepoRootForImportPath(path, get.IgnoreMod, security)
@ -397,11 +408,22 @@ func (l *loggingRepo) Zip(dst io.Writer, version string) error {
}
// A notExistError is like os.ErrNotExist, but with a custom message
type notExistError string
type notExistError struct {
err error
}
func notExistErrorf(format string, args ...interface{}) error {
return notExistError{fmt.Errorf(format, args...)}
}
func (e notExistError) Error() string {
return string(e)
return e.err.Error()
}
func (notExistError) Is(target error) bool {
return target == os.ErrNotExist
}
func (e notExistError) Unwrap() error {
return e.err
}

14
libgo/go/cmd/go/internal/modfetch/sumdb.go
View file

@ -24,11 +24,11 @@ import (
"cmd/go/internal/cfg"
"cmd/go/internal/get"
"cmd/go/internal/lockedfile"
"cmd/go/internal/module"
"cmd/go/internal/note"
"cmd/go/internal/str"
"cmd/go/internal/sumweb"
"cmd/go/internal/web"
"golang.org/x/mod/module"
"golang.org/x/mod/sumdb"
"golang.org/x/mod/sumdb/note"
)
// useSumDB reports whether to use the Go checksum database for the given module.
@ -52,11 +52,11 @@ func lookupSumDB(mod module.Version) (dbname string, lines []string, err error)
var (
dbOnce sync.Once
dbName string
db *sumweb.Conn
db *sumdb.Client
dbErr error
)
func dbDial() (dbName string, db *sumweb.Conn, err error) {
func dbDial() (dbName string, db *sumdb.Client, err error) {
// $GOSUMDB can be "key" or "key url",
// and the key can be a full verifier key
// or a host on our list of known keys.
@ -106,7 +106,7 @@ func dbDial() (dbName string, db *sumweb.Conn, err error) {
base = u
}
return name, sumweb.NewConn(&dbClient{key: key[0], name: name, direct: direct, base: base}), nil
return name, sumdb.NewClient(&dbClient{key: key[0], name: name, direct: direct, base: base}), nil
}
type dbClient struct {
@ -227,7 +227,7 @@ func (*dbClient) WriteConfig(file string, old, new []byte) error {
return err
}
if len(data) > 0 && !bytes.Equal(data, old) {
return sumweb.ErrWriteConflict
return sumdb.ErrWriteConflict
}
if _, err := f.Seek(0, 0); err != nil {
return err

173
libgo/go/cmd/go/internal/modfetch/unzip.go
View file

@ -1,173 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package modfetch
import (
"archive/zip"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/module"
"cmd/go/internal/str"
)
func Unzip(dir, zipfile, prefix string, maxSize int64) error {
// TODO(bcmills): The maxSize parameter is invariantly 0. Remove it.
if maxSize == 0 {
maxSize = codehost.MaxZipFile
}
// Directory can exist, but must be empty.
files, _ := ioutil.ReadDir(dir)
if len(files) > 0 {
return fmt.Errorf("target directory %v exists and is not empty", dir)
}
if err := os.MkdirAll(dir, 0777); err != nil {
return err
}
f, err := os.Open(zipfile)
if err != nil {
return err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return err
}
z, err := zip.NewReader(f, info.Size())
if err != nil {
return fmt.Errorf("unzip %v: %s", zipfile, err)
}
foldPath := make(map[string]string)
var checkFold func(string) error
checkFold = func(name string) error {
fold := str.ToFold(name)
if foldPath[fold] == name {
return nil
}
dir := path.Dir(name)
if dir != "." {
if err := checkFold(dir); err != nil {
return err
}
}
if foldPath[fold] == "" {
foldPath[fold] = name
return nil
}
other := foldPath[fold]
return fmt.Errorf("unzip %v: case-insensitive file name collision: %q and %q", zipfile, other, name)
}
// Check total size, valid file names.
var size int64
for _, zf := range z.File {
if !str.HasPathPrefix(zf.Name, prefix) {
return fmt.Errorf("unzip %v: unexpected file name %s", zipfile, zf.Name)
}
if zf.Name == prefix || strings.HasSuffix(zf.Name, "/") {
continue
}
name := zf.Name[len(prefix)+1:]
if err := module.CheckFilePath(name); err != nil {
return fmt.Errorf("unzip %v: %v", zipfile, err)
}
if err := checkFold(name); err != nil {
return err
}
if path.Clean(zf.Name) != zf.Name || strings.HasPrefix(zf.Name[len(prefix)+1:], "/") {
return fmt.Errorf("unzip %v: invalid file name %s", zipfile, zf.Name)
}
s := int64(zf.UncompressedSize64)
if s < 0 || maxSize-size < s {
return fmt.Errorf("unzip %v: content too large", zipfile)
}
size += s
}
// Unzip, enforcing sizes checked earlier.
for _, zf := range z.File {
if zf.Name == prefix || strings.HasSuffix(zf.Name, "/") {
continue
}
name := zf.Name[len(prefix):]
dst := filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(dst), 0777); err != nil {
return err
}
w, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0444)
if err != nil {
return fmt.Errorf("unzip %v: %v", zipfile, err)
}
r, err := zf.Open()
if err != nil {
w.Close()
return fmt.Errorf("unzip %v: %v", zipfile, err)
}
lr := &io.LimitedReader{R: r, N: int64(zf.UncompressedSize64) + 1}
_, err = io.Copy(w, lr)
r.Close()
if err != nil {
w.Close()
return fmt.Errorf("unzip %v: %v", zipfile, err)
}
if err := w.Close(); err != nil {
return fmt.Errorf("unzip %v: %v", zipfile, err)
}
if lr.N <= 0 {
return fmt.Errorf("unzip %v: content too large", zipfile)
}
}
return nil
}
// makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
// and its transitive contents.
func makeDirsReadOnly(dir string) {
type pathMode struct {
path string
mode os.FileMode
}
var dirs []pathMode // in lexical order
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err == nil && info.Mode()&0222 != 0 {
if info.IsDir() {
dirs = append(dirs, pathMode{path, info.Mode()})
}
}
return nil
})
// Run over list backward to chmod children before parents.
for i := len(dirs) - 1; i >= 0; i-- {
os.Chmod(dirs[i].path, dirs[i].mode&^0222)
}
}
// RemoveAll removes a directory written by Download or Unzip, first applying
// any permission changes needed to do so.
func RemoveAll(dir string) error {
// Module cache has 0555 directories; make them writable in order to remove content.
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil // ignore errors walking in file system
}
if info.IsDir() {
os.Chmod(path, 0777)
}
return nil
})
return os.RemoveAll(dir)
}

2124
libgo/go/cmd/go/internal/modfetch/zip_sum_test/testdata/zip_sums.csv vendored Normal file
View file

File diff suppressed because it is too large Load diff

230
libgo/go/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go Normal file
View file

@ -0,0 +1,230 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package zip_sum_test tests that the module zip files produced by modfetch
// have consistent content sums. Ideally the zip files themselves are also
// stable over time, though this is not strictly necessary.
//
// This test loads a table from testdata/zip_sums.csv. The table has columns
// for module path, version, content sum, and zip file hash. The table
// includes a large number of real modules. The test downloads these modules
// in direct mode and verifies the zip files.
//
// This test is very slow, and it depends on outside modules that change
// frequently, so this is a manual test. To enable it, pass the -zipsum flag.
package zip_sum_test
import (
"crypto/sha256"
"encoding/csv"
"encoding/hex"
"flag"
"fmt"
"internal/testenv"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"cmd/go/internal/cfg"
"cmd/go/internal/modfetch"
"cmd/go/internal/modload"
"golang.org/x/mod/module"
)
var (
updateTestData = flag.Bool("u", false, "when set, tests may update files in testdata instead of failing")
enableZipSum = flag.Bool("zipsum", false, "enable TestZipSums")
debugZipSum = flag.Bool("testwork", false, "when set, TestZipSums will preserve its test directory")
modCacheDir = flag.String("zipsumcache", "", "module cache to use instead of temp directory")
shardCount = flag.Int("zipsumshardcount", 1, "number of shards to divide TestZipSums into")
shardIndex = flag.Int("zipsumshard", 0, "index of TestZipSums shard to test (0 <= zipsumshard < zipsumshardcount)")
)
const zipSumsPath = "testdata/zip_sums.csv"
type zipSumTest struct {
m module.Version
wantSum, wantFileHash string
}
func TestZipSums(t *testing.T) {
if !*enableZipSum {
// This test is very slow and heavily dependent on external repositories.
// Only run it explicitly.
t.Skip("TestZipSum not enabled with -zipsum")
}
if *shardCount < 1 {
t.Fatal("-zipsumshardcount must be a positive integer")
}
if *shardIndex < 0 || *shardCount <= *shardIndex {
t.Fatal("-zipsumshard must be between 0 and -zipsumshardcount")
}
testenv.MustHaveGoBuild(t)
testenv.MustHaveExternalNetwork(t)
testenv.MustHaveExecPath(t, "bzr")
testenv.MustHaveExecPath(t, "git")
// TODO(jayconrod): add hg, svn, and fossil modules to testdata.
// Could not find any for now.
tests, err := readZipSumTests()
if err != nil {
t.Fatal(err)
}
if *modCacheDir != "" {
cfg.BuildContext.GOPATH = *modCacheDir
} else {
tmpDir, err := ioutil.TempDir("", "TestZipSums")
if err != nil {
t.Fatal(err)
}
if *debugZipSum {
fmt.Fprintf(os.Stderr, "TestZipSums: modCacheDir: %s\n", tmpDir)
} else {
defer os.RemoveAll(tmpDir)
}
cfg.BuildContext.GOPATH = tmpDir
}
cfg.GOPROXY = "direct"
cfg.GOSUMDB = "off"
modload.Init()
// Shard tests by downloading only every nth module when shard flags are set.
// This makes it easier to test small groups of modules quickly. We avoid
// testing similarly named modules together (the list is sorted by module
// path and version).
if *shardCount > 1 {
r := *shardIndex
w := 0
for r < len(tests) {
tests[w] = tests[r]
w++
r += *shardCount
}
tests = tests[:w]
}
// Download modules with a rate limit. We may run out of file descriptors
// or cause timeouts without a limit.
needUpdate := false
for i := range tests {
test := &tests[i]
name := fmt.Sprintf("%s@%s", strings.ReplaceAll(test.m.Path, "/", "_"), test.m.Version)
t.Run(name, func(t *testing.T) {
t.Parallel()
zipPath, err := modfetch.DownloadZip(test.m)
if err != nil {
if *updateTestData {
t.Logf("%s: could not download module: %s (will remove from testdata)", test.m, err)
test.m.Path = "" // mark for deletion
needUpdate = true
} else {
t.Errorf("%s: could not download mdoule: %s", test.m, err)
}
return
}
sum := modfetch.Sum(test.m)
if sum != test.wantSum {
if *updateTestData {
t.Logf("%s: updating content sum to %s", test.m, sum)
test.wantSum = sum
needUpdate = true
} else {
t.Errorf("%s: got content sum %s; want sum %s", test.m, sum, test.wantSum)
return
}
}
h := sha256.New()
f, err := os.Open(zipPath)
if err != nil {
t.Errorf("%s: %v", test.m, err)
}
defer f.Close()
if _, err := io.Copy(h, f); err != nil {
t.Errorf("%s: %v", test.m, err)
}
zipHash := hex.EncodeToString(h.Sum(nil))
if zipHash != test.wantFileHash {
if *updateTestData {
t.Logf("%s: updating zip file hash to %s", test.m, zipHash)
test.wantFileHash = zipHash
needUpdate = true
} else {
t.Errorf("%s: got zip file hash %s; want hash %s (but content sum matches)", test.m, zipHash, test.wantFileHash)
}
}
})
}
if needUpdate {
// Remove tests marked for deletion
r, w := 0, 0
for r < len(tests) {
if tests[r].m.Path != "" {
tests[w] = tests[r]
w++
}
r++
}
tests = tests[:w]
if err := writeZipSumTests(tests); err != nil {
t.Error(err)
}
}
}
func readZipSumTests() ([]zipSumTest, error) {
f, err := os.Open(filepath.FromSlash(zipSumsPath))
if err != nil {
return nil, err
}
defer f.Close()
r := csv.NewReader(f)
var tests []zipSumTest
for {
line, err := r.Read()
if err == io.EOF {
break
} else if err != nil {
return nil, err
} else if len(line) != 4 {
return nil, fmt.Errorf("%s:%d: malformed line", f.Name(), len(tests)+1)
}
test := zipSumTest{m: module.Version{Path: line[0], Version: line[1]}, wantSum: line[2], wantFileHash: line[3]}
tests = append(tests, test)
}
return tests, nil
}
func writeZipSumTests(tests []zipSumTest) (err error) {
f, err := os.Create(filepath.FromSlash(zipSumsPath))
if err != nil {
return err
}
defer func() {
if cerr := f.Close(); err == nil && cerr != nil {
err = cerr
}
}()
w := csv.NewWriter(f)
line := make([]string, 0, 4)
for _, test := range tests {
line = append(line[:0], test.m.Path, test.m.Version, test.wantSum, test.wantFileHash)
if err := w.Write(line); err != nil {
return err
}
}
w.Flush()
return nil
}

47
libgo/go/cmd/go/internal/modfile/gopkgin.go
View file

@ -1,47 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// TODO: Figure out what gopkg.in should do.
package modfile
import "strings"
// ParseGopkgIn splits gopkg.in import paths into their constituent parts
func ParseGopkgIn(path string) (root, repo, major, subdir string, ok bool) {
if !strings.HasPrefix(path, "gopkg.in/") {
return
}
f := strings.Split(path, "/")
if len(f) >= 2 {
if elem, v, ok := dotV(f[1]); ok {
root = strings.Join(f[:2], "/")
repo = "github.com/go-" + elem + "/" + elem
major = v
subdir = strings.Join(f[2:], "/")
return root, repo, major, subdir, true
}
}
if len(f) >= 3 {
if elem, v, ok := dotV(f[2]); ok {
root = strings.Join(f[:3], "/")
repo = "github.com/" + f[1] + "/" + elem
major = v
subdir = strings.Join(f[3:], "/")
return root, repo, major, subdir, true
}
}
return
}
func dotV(name string) (elem, v string, ok bool) {
i := len(name) - 1
for i >= 0 && '0' <= name[i] && name[i] <= '9' {
i--
}
if i <= 2 || i+1 >= len(name) || name[i-1] != '.' || name[i] != 'v' || name[i+1] == '0' && len(name) != i+2 {
return "", "", false
}
return name[:i-1], name[i:], true
}

365
libgo/go/cmd/go/internal/modfile/read_test.go
View file

@ -1,365 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package modfile
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"testing"
)
// exists reports whether the named file exists.
func exists(name string) bool {
_, err := os.Stat(name)
return err == nil
}
// Test that reading and then writing the golden files
// does not change their output.
func TestPrintGolden(t *testing.T) {
outs, err := filepath.Glob("testdata/*.golden")
if err != nil {
t.Fatal(err)
}
for _, out := range outs {
testPrint(t, out, out)
}
}
// testPrint is a helper for testing the printer.
// It reads the file named in, reformats it, and compares
// the result to the file named out.
func testPrint(t *testing.T, in, out string) {
data, err := ioutil.ReadFile(in)
if err != nil {
t.Error(err)
return
}
golden, err := ioutil.ReadFile(out)
if err != nil {
t.Error(err)
return
}
base := "testdata/" + filepath.Base(in)
f, err := parse(in, data)
if err != nil {
t.Error(err)
return
}
ndata := Format(f)
if !bytes.Equal(ndata, golden) {
t.Errorf("formatted %s incorrectly: diff shows -golden, +ours", base)
tdiff(t, string(golden), string(ndata))
return
}
}
func TestParseLax(t *testing.T) {
badFile := []byte(`module m
surprise attack
x y (
z
)
exclude v1.2.3
replace <-!!!
`)
_, err := ParseLax("file", badFile, nil)
if err != nil {
t.Fatalf("ParseLax did not ignore irrelevant errors: %v", err)
}
}
// Test that when files in the testdata directory are parsed
// and printed and parsed again, we get the same parse tree
// both times.
func TestPrintParse(t *testing.T) {
outs, err := filepath.Glob("testdata/*")
if err != nil {
t.Fatal(err)
}
for _, out := range outs {
data, err := ioutil.ReadFile(out)
if err != nil {
t.Error(err)
continue
}
base := "testdata/" + filepath.Base(out)
f, err := parse(base, data)
if err != nil {
t.Errorf("parsing original: %v", err)
continue
}
ndata := Format(f)
f2, err := parse(base, ndata)
if err != nil {
t.Errorf("parsing reformatted: %v", err)
continue
}
eq := eqchecker{file: base}
if err := eq.check(f, f2); err != nil {
t.Errorf("not equal (parse/Format/parse): %v", err)
}
pf1, err := Parse(base, data, nil)
if err != nil {
switch base {
case "testdata/replace2.in", "testdata/gopkg.in.golden":
t.Errorf("should parse %v: %v", base, err)
}
}
if err == nil {
pf2, err := Parse(base, ndata, nil)
if err != nil {
t.Errorf("Parsing reformatted: %v", err)
continue
}
eq := eqchecker{file: base}
if err := eq.check(pf1, pf2); err != nil {
t.Errorf("not equal (parse/Format/Parse): %v", err)
}
ndata2, err := pf1.Format()
if err != nil {
t.Errorf("reformat: %v", err)
}
pf3, err := Parse(base, ndata2, nil)
if err != nil {
t.Errorf("Parsing reformatted2: %v", err)
continue
}
eq = eqchecker{file: base}
if err := eq.check(pf1, pf3); err != nil {
t.Errorf("not equal (Parse/Format/Parse): %v", err)
}
ndata = ndata2
}
if strings.HasSuffix(out, ".in") {
golden, err := ioutil.ReadFile(strings.TrimSuffix(out, ".in") + ".golden")
if err != nil {
t.Error(err)
continue
}
if !bytes.Equal(ndata, golden) {
t.Errorf("formatted %s incorrectly: diff shows -golden, +ours", base)
tdiff(t, string(golden), string(ndata))
return
}
}
}
}
// An eqchecker holds state for checking the equality of two parse trees.
type eqchecker struct {
file string
pos Position
}
// errorf returns an error described by the printf-style format and arguments,
// inserting the current file position before the error text.
func (eq *eqchecker) errorf(format string, args ...interface{}) error {
return fmt.Errorf("%s:%d: %s", eq.file, eq.pos.Line,
fmt.Sprintf(format, args...))
}
// check checks that v and w represent the same parse tree.
// If not, it returns an error describing the first difference.
func (eq *eqchecker) check(v, w interface{}) error {
return eq.checkValue(reflect.ValueOf(v), reflect.ValueOf(w))
}
var (
posType = reflect.TypeOf(Position{})
commentsType = reflect.TypeOf(Comments{})
)
// checkValue checks that v and w represent the same parse tree.
// If not, it returns an error describing the first difference.
func (eq *eqchecker) checkValue(v, w reflect.Value) error {
// inner returns the innermost expression for v.
// if v is a non-nil interface value, it returns the concrete
// value in the interface.
inner := func(v reflect.Value) reflect.Value {
for {
if v.Kind() == reflect.Interface && !v.IsNil() {
v = v.Elem()
continue
}
break
}
return v
}
v = inner(v)
w = inner(w)
if v.Kind() == reflect.Invalid && w.Kind() == reflect.Invalid {
return nil
}
if v.Kind() == reflect.Invalid {
return eq.errorf("nil interface became %s", w.Type())
}
if w.Kind() == reflect.Invalid {
return eq.errorf("%s became nil interface", v.Type())
}
if v.Type() != w.Type() {
return eq.errorf("%s became %s", v.Type(), w.Type())
}
if p, ok := v.Interface().(Expr); ok {
eq.pos, _ = p.Span()
}
switch v.Kind() {
default:
return eq.errorf("unexpected type %s", v.Type())
case reflect.Bool, reflect.Int, reflect.String:
vi := v.Interface()
wi := w.Interface()
if vi != wi {
return eq.errorf("%v became %v", vi, wi)
}
case reflect.Slice:
vl := v.Len()
wl := w.Len()
for i := 0; i < vl || i < wl; i++ {
if i >= vl {
return eq.errorf("unexpected %s", w.Index(i).Type())
}
if i >= wl {
return eq.errorf("missing %s", v.Index(i).Type())
}
if err := eq.checkValue(v.Index(i), w.Index(i)); err != nil {
return err
}
}
case reflect.Struct:
// Fields in struct must match.
t := v.Type()
n := t.NumField()
for i := 0; i < n; i++ {
tf := t.Field(i)
switch {
default:
if err := eq.checkValue(v.Field(i), w.Field(i)); err != nil {
return err
}
case tf.Type == posType: // ignore positions
case tf.Type == commentsType: // ignore comment assignment
}
}
case reflect.Ptr, reflect.Interface:
if v.IsNil() != w.IsNil() {
if v.IsNil() {
return eq.errorf("unexpected %s", w.Elem().Type())
}
return eq.errorf("missing %s", v.Elem().Type())
}
if err := eq.checkValue(v.Elem(), w.Elem()); err != nil {
return err
}
}
return nil
}
// diff returns the output of running diff on b1 and b2.
func diff(b1, b2 []byte) (data []byte, err error) {
f1, err := ioutil.TempFile("", "testdiff")
if err != nil {
return nil, err
}
defer os.Remove(f1.Name())
defer f1.Close()
f2, err := ioutil.TempFile("", "testdiff")
if err != nil {
return nil, err
}
defer os.Remove(f2.Name())
defer f2.Close()
f1.Write(b1)
f2.Write(b2)
data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
if len(data) > 0 {
// diff exits with a non-zero status when the files don't match.
// Ignore that failure as long as we get output.
err = nil
}
return
}
// tdiff logs the diff output to t.Error.
func tdiff(t *testing.T, a, b string) {
data, err := diff([]byte(a), []byte(b))
if err != nil {
t.Error(err)
return
}
t.Error(string(data))
}
var modulePathTests = []struct {
input []byte
expected string
}{
{input: []byte("module \"github.com/rsc/vgotest\""), expected: "github.com/rsc/vgotest"},
{input: []byte("module github.com/rsc/vgotest"), expected: "github.com/rsc/vgotest"},
{input: []byte("module \"github.com/rsc/vgotest\""), expected: "github.com/rsc/vgotest"},
{input: []byte("module github.com/rsc/vgotest"), expected: "github.com/rsc/vgotest"},
{input: []byte("module `github.com/rsc/vgotest`"), expected: "github.com/rsc/vgotest"},
{input: []byte("module \"github.com/rsc/vgotest/v2\""), expected: "github.com/rsc/vgotest/v2"},
{input: []byte("module github.com/rsc/vgotest/v2"), expected: "github.com/rsc/vgotest/v2"},
{input: []byte("module \"gopkg.in/yaml.v2\""), expected: "gopkg.in/yaml.v2"},
{input: []byte("module gopkg.in/yaml.v2"), expected: "gopkg.in/yaml.v2"},
{input: []byte("module \"gopkg.in/check.v1\"\n"), expected: "gopkg.in/check.v1"},
{input: []byte("module \"gopkg.in/check.v1\n\""), expected: ""},
{input: []byte("module gopkg.in/check.v1\n"), expected: "gopkg.in/check.v1"},
{input: []byte("module \"gopkg.in/check.v1\"\r\n"), expected: "gopkg.in/check.v1"},
{input: []byte("module gopkg.in/check.v1\r\n"), expected: "gopkg.in/check.v1"},
{input: []byte("module \"gopkg.in/check.v1\"\n\n"), expected: "gopkg.in/check.v1"},
{input: []byte("module gopkg.in/check.v1\n\n"), expected: "gopkg.in/check.v1"},
{input: []byte("module \n\"gopkg.in/check.v1\"\n\n"), expected: ""},
{input: []byte("module \ngopkg.in/check.v1\n\n"), expected: ""},
{input: []byte("module \"gopkg.in/check.v1\"asd"), expected: ""},
{input: []byte("module \n\"gopkg.in/check.v1\"\n\n"), expected: ""},
{input: []byte("module \ngopkg.in/check.v1\n\n"), expected: ""},
{input: []byte("module \"gopkg.in/check.v1\"asd"), expected: ""},
{input: []byte("module \nmodule a/b/c "), expected: "a/b/c"},
{input: []byte("module \" \""), expected: " "},
{input: []byte("module "), expected: ""},
{input: []byte("module \" a/b/c \""), expected: " a/b/c "},
{input: []byte("module \"github.com/rsc/vgotest1\" // with a comment"), expected: "github.com/rsc/vgotest1"},
}
func TestModulePath(t *testing.T) {
for _, test := range modulePathTests {
t.Run(string(test.input), func(t *testing.T) {
result := ModulePath(test.input)
if result != test.expected {
t.Fatalf("ModulePath(%q): %s, want %s", string(test.input), result, test.expected)
}
})
}
}

90
libgo/go/cmd/go/internal/modfile/rule_test.go
View file

@ -1,90 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package modfile
import (
"bytes"
"fmt"
"testing"
)
var addRequireTests = []struct {
in string
path string
vers string
out string
}{
{
`
module m
require x.y/z v1.2.3
`,
"x.y/z", "v1.5.6",
`
module m
require x.y/z v1.5.6
`,
},
{
`
module m
require x.y/z v1.2.3
`,
"x.y/w", "v1.5.6",
`
module m
require (
x.y/z v1.2.3
x.y/w v1.5.6
)
`,
},
{
`
module m
require x.y/z v1.2.3
require x.y/q/v2 v2.3.4
`,
"x.y/w", "v1.5.6",
`
module m
require x.y/z v1.2.3
require (
x.y/q/v2 v2.3.4
x.y/w v1.5.6
)
`,
},
}
func TestAddRequire(t *testing.T) {
for i, tt := range addRequireTests {
t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
f, err := Parse("in", []byte(tt.in), nil)
if err != nil {
t.Fatal(err)
}
g, err := Parse("out", []byte(tt.out), nil)
if err != nil {
t.Fatal(err)
}
golden, err := g.Format()
if err != nil {
t.Fatal(err)
}
if err := f.AddRequire(tt.path, tt.vers); err != nil {
t.Fatal(err)
}
out, err := f.Format()
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(out, golden) {
t.Errorf("have:\n%s\nwant:\n%s", out, golden)
}
})
}
}

29
libgo/go/cmd/go/internal/modfile/testdata/block.golden vendored
View file

@ -1,29 +0,0 @@
// comment
x "y" z
// block
block ( // block-eol
// x-before-line
"x" ( y // x-eol
"x1"
"x2"
// line
"x3"
"x4"
"x5"
// y-line
"y" // y-eol
"z" // z-eol
) // block-eol2
block2 (
x
y
z
)
// eof

29
libgo/go/cmd/go/internal/modfile/testdata/block.in vendored
View file

@ -1,29 +0,0 @@
// comment
x "y" z
// block
block ( // block-eol
// x-before-line
"x" ( y // x-eol
"x1"
"x2"
// line
"x3"
"x4"
"x5"
// y-line
"y" // y-eol
"z" // z-eol
) // block-eol2
block2 (x
y
z
)
// eof

10
libgo/go/cmd/go/internal/modfile/testdata/comment.golden vendored
View file

@ -1,10 +0,0 @@
// comment
module "x" // eol
// mid comment
// comment 2
// comment 2 line 2
module "y" // eoy
// comment 3

8
libgo/go/cmd/go/internal/modfile/testdata/comment.in vendored
View file

@ -1,8 +0,0 @@
// comment
module "x" // eol
// mid comment
// comment 2
// comment 2 line 2
module "y" // eoy
// comment 3

0
libgo/go/cmd/go/internal/modfile/testdata/empty.golden vendored
View file

0
libgo/go/cmd/go/internal/modfile/testdata/empty.in vendored
View file

6
libgo/go/cmd/go/internal/modfile/testdata/gopkg.in.golden vendored
View file

@ -1,6 +0,0 @@
module x
require (
gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528
gopkg.in/yaml.v2 v2.2.1
)

Some files were not shown because too many files have changed in this diff Show more