FreeChainXenon/gcc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

libgo: update to Go1.17rc2

Browse source 
Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/341629
This commit is contained in:
Ian Lance Taylor 2021-07-30 14:28:58 -07:00
parent 72be20e202
commit c5b21c3f4c
2004 changed files with 74916 additions and 23077 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
gcc/go/gofrontend/MERGE
View file

@ -1,4 +1,4 @@
5edbb624b2595d644eb6842c952a292c41f7d6fa
33f65dce43bd01c1fa38cd90a78c9aea6ca6dd59
The first line of this file holds the git revision number of the last
merge done from the gofrontend repository.

2
libgo/MERGE
View file

@ -1,4 +1,4 @@
7677616a263e8ded606cc8297cb67ddc667a876e
72ab3ff68b1ec894fe5599ec82b8849f3baa9d94
The first line of this file holds the git revision number of the
last merge done from the master library sources.

64
libgo/Makefile.am
View file

@ -366,6 +366,7 @@ toolexeclibgoregexp_DATA = \
toolexeclibgoruntimedir = $(toolexeclibgodir)/runtime
toolexeclibgoruntime_DATA = \
runtime/cgo.gox \
runtime/debug.gox \
runtime/metrics.gox \
runtime/pprof.gox \
@ -428,7 +429,9 @@ noinst_DATA = \
internal/testenv.gox \
internal/trace.gox \
net/internal/socktest.gox \
os/signal/internal/pty.gox
os/signal/internal/pty.gox \
reflect/internal/example1.gox \
reflect/internal/example2.gox
if LIBGO_IS_RTEMS
rtems_task_variable_add_file = runtime/rtems-task-variable-add.c
@ -480,14 +483,10 @@ version.go: s-version; @true
s-version: Makefile
rm -f version.go.tmp
echo "package sys" > version.go.tmp
echo 'func init() { DefaultGoroot = "$(prefix)" }' >> version.go.tmp
echo 'const TheVersion = "'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'"' >> version.go.tmp
echo 'const Goexperiment = ``' >> version.go.tmp
echo 'const GOARCH = "'$(GOARCH)'"' >> version.go.tmp
echo 'const GOOS = "'$(GOOS)'"' >> version.go.tmp
echo 'const GccgoToolDir = "$(libexecsubdir)"' >> version.go.tmp
echo >> version.go.tmp
echo "type ArchFamilyType int" >> version.go.tmp
echo 'const StackGuardMultiplierDefault = 1' >> version.go.tmp
echo >> version.go.tmp
echo "const (" >> version.go.tmp
echo " UNKNOWN ArchFamilyType = iota" >> version.go.tmp
@ -507,13 +506,13 @@ s-version: Makefile
done
echo >> version.go.tmp
echo "const (" >> version.go.tmp
echo " ArchFamily = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) family`" >> version.go.tmp
echo " BigEndian = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) bigendian`" >> version.go.tmp
echo " CacheLineSize = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) cachelinesize`" >> version.go.tmp
echo " DefaultPhysPageSize = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) defaultphyspagesize`" >> version.go.tmp
echo " Int64Align = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) int64align`" >> version.go.tmp
echo " MinFrameSize = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) minframesize`" >> version.go.tmp
echo " PCQuantum = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) pcquantum`" >> version.go.tmp
echo " _ArchFamily = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) family`" >> version.go.tmp
echo " _BigEndian = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) bigendian`" >> version.go.tmp
echo " _DefaultPhysPageSize = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) defaultphyspagesize`" >> version.go.tmp
echo " _Int64Align = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) int64align`" >> version.go.tmp
echo " _MinFrameSize = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) minframesize`" >> version.go.tmp
echo " _PCQuantum = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) pcquantum`" >> version.go.tmp
echo " _StackAlign = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) stackalign`" >> version.go.tmp
echo ")" >> version.go.tmp
echo >> version.go.tmp
for a in $(ALLGOOS); do \
@ -526,7 +525,6 @@ s-version: Makefile
fi; \
done
echo >> version.go.tmp
echo "type Uintreg uintptr" >> version.go.tmp
$(SHELL) $(srcdir)/mvifdiff.sh version.go.tmp version.go
$(STAMP) $@
@ -547,24 +545,31 @@ s-gcpu: Makefile
$(SHELL) $(srcdir)/mvifdiff.sh gcpugen.go.tmp gcpugen.go
$(STAMP) $@
buildcfg.go: s-buildcfg; @true
s-buildcfg: Makefile
rm -f buildcfg.go.tmp
echo "package buildcfg" > buildcfg.go.tmp
echo "import \"runtime\"" >> buildcfg.go.tmp
echo 'func defaultGOROOTValue() string { return `$(prefix)` }' >> buildcfg.go.tmp
echo 'const defaultGO386 = `sse2`' >> buildcfg.go.tmp
echo 'const defaultGOARM = `5`' >> buildcfg.go.tmp
echo 'const defaultGOMIPS = `hardfloat`' >> buildcfg.go.tmp
echo 'const defaultGOMIPS64 = `hardfloat`' >> buildcfg.go.tmp
echo 'const defaultGOPPC64 = `power8`' >> buildcfg.go.tmp
echo 'const defaultGOEXPERIMENT = `fieldtrack`' >> buildcfg.go.tmp
echo 'const defaultGO_EXTLINK_ENABLED = ``' >> buildcfg.go.tmp
echo 'const defaultGO_LDSO = ``' >> buildcfg.go.tmp
echo 'const version = `'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'`' >> buildcfg.go.tmp
echo 'const defaultGOOS = runtime.GOOS' >> buildcfg.go.tmp
echo 'const defaultGOARCH = runtime.GOARCH' >> buildcfg.go.tmp
$(SHELL) $(srcdir)/mvifdiff.sh buildcfg.go.tmp buildcfg.go
$(STAMP) $@
objabi.go: s-objabi; @true
s-objabi: Makefile
rm -f objabi.go.tmp
echo "package objabi" > objabi.go.tmp
echo "import \"runtime\"" >> objabi.go.tmp
echo 'func defaultGOROOTValue() string { return `$(prefix)` }' >> objabi.go.tmp
echo 'const defaultGO386 = `sse2`' >> objabi.go.tmp
echo 'const defaultGOARM = `5`' >> objabi.go.tmp
echo 'const defaultGOMIPS = `hardfloat`' >> objabi.go.tmp
echo 'const defaultGOMIPS64 = `hardfloat`' >> objabi.go.tmp
echo 'const defaultGOPPC64 = `power8`' >> objabi.go.tmp
echo 'const defaultGOOS = runtime.GOOS' >> objabi.go.tmp
echo 'const defaultGOARCH = runtime.GOARCH' >> objabi.go.tmp
echo 'const defaultGO_EXTLINK_ENABLED = ``' >> objabi.go.tmp
echo 'const defaultGO_LDSO = ``' >> objabi.go.tmp
echo 'const version = `'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'`' >> objabi.go.tmp
echo 'const stackGuardMultiplierDefault = 1' >> objabi.go.tmp
echo 'const goexperiment = ``' >> objabi.go.tmp
$(SHELL) $(srcdir)/mvifdiff.sh objabi.go.tmp objabi.go
$(STAMP) $@
@ -671,7 +676,7 @@ s-zstdpkglist: Makefile libgo-packages.txt
echo 'package goroot' > zstdpkglist.go.tmp
echo "" >> zstdpkglist.go.tmp
echo 'var stdpkg = map[string]bool{' >> zstdpkglist.go.tmp
echo $(libgo_go_objs) 'unsafe.lo' 'runtime/cgo.lo' | sed 's|[a-z0-9_./]*_c\.lo||g' | sed 's|golang\.org/[a-z0-9_./]*\.lo||g' | sed 's|\([a-z0-9_./]*\)\.lo|"\1": true,|g' >> zstdpkglist.go.tmp
echo $(libgo_go_objs) 'unsafe.lo' | sed 's|[a-z0-9_./]*_c\.lo||g' | sed 's|golang\.org/[a-z0-9_./]*\.lo||g' | sed 's|\([a-z0-9_./]*\)\.lo|"\1": true,|g' >> zstdpkglist.go.tmp
echo '}' >> zstdpkglist.go.tmp
$(SHELL) $(srcdir)/mvifdiff.sh zstdpkglist.go.tmp zstdpkglist.go
$(STAMP) $@
@ -1054,6 +1059,9 @@ internal/cpu.lo.dep: $(extra_go_files_internal_cpu)
extra_go_files_golang_org_x_sys_cpu = gcpugen.go
golang.org/x/sys/cpu.lo.dep: $(extra_go_files_golang_org_x_sys_cpu)
extra_go_files_internal_buildcfg = buildcfg.go
cmd/internal/buildcfg.lo.dep: $(extra_go_files_internal_buildcfg)
extra_go_files_internal_goroot = zstdpkglist.go
internal/goroot.lo.dep: $(extra_go_files_internal_goroot)

60
libgo/Makefile.in
View file

@ -839,6 +839,7 @@ toolexeclibgoregexp_DATA = \
toolexeclibgoruntimedir = $(toolexeclibgodir)/runtime
toolexeclibgoruntime_DATA = \
runtime/cgo.gox \
runtime/debug.gox \
runtime/metrics.gox \
runtime/pprof.gox \
@ -892,6 +893,7 @@ noinst_DATA = golang.org/x/net/nettest.gox internal/cfg.gox \
internal/obscuretestdata.gox internal/profile.gox \
internal/testenv.gox internal/trace.gox \
net/internal/socktest.gox os/signal/internal/pty.gox \
reflect/internal/example1.gox reflect/internal/example2.gox \
zdefaultcc.go
@LIBGO_IS_RTEMS_FALSE@rtems_task_variable_add_file =
@LIBGO_IS_RTEMS_TRUE@rtems_task_variable_add_file = runtime/rtems-task-variable-add.c
@ -1135,6 +1137,7 @@ runtime_pprof_check_GOCFLAGS = -static-libgo -fno-inline
extra_go_files_runtime_internal_sys = version.go
extra_go_files_internal_cpu = cpugen.go
extra_go_files_golang_org_x_sys_cpu = gcpugen.go
extra_go_files_internal_buildcfg = buildcfg.go
extra_go_files_internal_goroot = zstdpkglist.go
extra_go_files_go_types = gccgosizes.go
extra_go_files_cmd_internal_objabi = objabi.go
@ -2687,14 +2690,10 @@ version.go: s-version; @true
s-version: Makefile
rm -f version.go.tmp
echo "package sys" > version.go.tmp
echo 'func init() { DefaultGoroot = "$(prefix)" }' >> version.go.tmp
echo 'const TheVersion = "'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'"' >> version.go.tmp
echo 'const Goexperiment = ``' >> version.go.tmp
echo 'const GOARCH = "'$(GOARCH)'"' >> version.go.tmp
echo 'const GOOS = "'$(GOOS)'"' >> version.go.tmp
echo 'const GccgoToolDir = "$(libexecsubdir)"' >> version.go.tmp
echo >> version.go.tmp
echo "type ArchFamilyType int" >> version.go.tmp
echo 'const StackGuardMultiplierDefault = 1' >> version.go.tmp
echo >> version.go.tmp
echo "const (" >> version.go.tmp
echo " UNKNOWN ArchFamilyType = iota" >> version.go.tmp
@ -2714,13 +2713,13 @@ s-version: Makefile
done
echo >> version.go.tmp
echo "const (" >> version.go.tmp
echo " ArchFamily = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) family`" >> version.go.tmp
echo " BigEndian = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) bigendian`" >> version.go.tmp
echo " CacheLineSize = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) cachelinesize`" >> version.go.tmp
echo " DefaultPhysPageSize = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) defaultphyspagesize`" >> version.go.tmp
echo " Int64Align = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) int64align`" >> version.go.tmp
echo " MinFrameSize = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) minframesize`" >> version.go.tmp
echo " PCQuantum = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) pcquantum`" >> version.go.tmp
echo " _ArchFamily = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) family`" >> version.go.tmp
echo " _BigEndian = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) bigendian`" >> version.go.tmp
echo " _DefaultPhysPageSize = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) defaultphyspagesize`" >> version.go.tmp
echo " _Int64Align = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) int64align`" >> version.go.tmp
echo " _MinFrameSize = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) minframesize`" >> version.go.tmp
echo " _PCQuantum = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) pcquantum`" >> version.go.tmp
echo " _StackAlign = `$(SHELL) $(srcdir)/goarch.sh $(GOARCH) stackalign`" >> version.go.tmp
echo ")" >> version.go.tmp
echo >> version.go.tmp
for a in $(ALLGOOS); do \
@ -2733,7 +2732,6 @@ s-version: Makefile
fi; \
done
echo >> version.go.tmp
echo "type Uintreg uintptr" >> version.go.tmp
$(SHELL) $(srcdir)/mvifdiff.sh version.go.tmp version.go
$(STAMP) $@
@ -2754,24 +2752,31 @@ s-gcpu: Makefile
$(SHELL) $(srcdir)/mvifdiff.sh gcpugen.go.tmp gcpugen.go
$(STAMP) $@
buildcfg.go: s-buildcfg; @true
s-buildcfg: Makefile
rm -f buildcfg.go.tmp
echo "package buildcfg" > buildcfg.go.tmp
echo "import \"runtime\"" >> buildcfg.go.tmp
echo 'func defaultGOROOTValue() string { return `$(prefix)` }' >> buildcfg.go.tmp
echo 'const defaultGO386 = `sse2`' >> buildcfg.go.tmp
echo 'const defaultGOARM = `5`' >> buildcfg.go.tmp
echo 'const defaultGOMIPS = `hardfloat`' >> buildcfg.go.tmp
echo 'const defaultGOMIPS64 = `hardfloat`' >> buildcfg.go.tmp
echo 'const defaultGOPPC64 = `power8`' >> buildcfg.go.tmp
echo 'const defaultGOEXPERIMENT = `fieldtrack`' >> buildcfg.go.tmp
echo 'const defaultGO_EXTLINK_ENABLED = ``' >> buildcfg.go.tmp
echo 'const defaultGO_LDSO = ``' >> buildcfg.go.tmp
echo 'const version = `'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'`' >> buildcfg.go.tmp
echo 'const defaultGOOS = runtime.GOOS' >> buildcfg.go.tmp
echo 'const defaultGOARCH = runtime.GOARCH' >> buildcfg.go.tmp
$(SHELL) $(srcdir)/mvifdiff.sh buildcfg.go.tmp buildcfg.go
$(STAMP) $@
objabi.go: s-objabi; @true
s-objabi: Makefile
rm -f objabi.go.tmp
echo "package objabi" > objabi.go.tmp
echo "import \"runtime\"" >> objabi.go.tmp
echo 'func defaultGOROOTValue() string { return `$(prefix)` }' >> objabi.go.tmp
echo 'const defaultGO386 = `sse2`' >> objabi.go.tmp
echo 'const defaultGOARM = `5`' >> objabi.go.tmp
echo 'const defaultGOMIPS = `hardfloat`' >> objabi.go.tmp
echo 'const defaultGOMIPS64 = `hardfloat`' >> objabi.go.tmp
echo 'const defaultGOPPC64 = `power8`' >> objabi.go.tmp
echo 'const defaultGOOS = runtime.GOOS' >> objabi.go.tmp
echo 'const defaultGOARCH = runtime.GOARCH' >> objabi.go.tmp
echo 'const defaultGO_EXTLINK_ENABLED = ``' >> objabi.go.tmp
echo 'const defaultGO_LDSO = ``' >> objabi.go.tmp
echo 'const version = `'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'`' >> objabi.go.tmp
echo 'const stackGuardMultiplierDefault = 1' >> objabi.go.tmp
echo 'const goexperiment = ``' >> objabi.go.tmp
$(SHELL) $(srcdir)/mvifdiff.sh objabi.go.tmp objabi.go
$(STAMP) $@
@ -2872,7 +2877,7 @@ s-zstdpkglist: Makefile libgo-packages.txt
echo 'package goroot' > zstdpkglist.go.tmp
echo "" >> zstdpkglist.go.tmp
echo 'var stdpkg = map[string]bool{' >> zstdpkglist.go.tmp
echo $(libgo_go_objs) 'unsafe.lo' 'runtime/cgo.lo' | sed 's|[a-z0-9_./]*_c\.lo||g' | sed 's|golang\.org/[a-z0-9_./]*\.lo||g' | sed 's|\([a-z0-9_./]*\)\.lo|"\1": true,|g' >> zstdpkglist.go.tmp
echo $(libgo_go_objs) 'unsafe.lo' | sed 's|[a-z0-9_./]*_c\.lo||g' | sed 's|golang\.org/[a-z0-9_./]*\.lo||g' | sed 's|\([a-z0-9_./]*\)\.lo|"\1": true,|g' >> zstdpkglist.go.tmp
echo '}' >> zstdpkglist.go.tmp
$(SHELL) $(srcdir)/mvifdiff.sh zstdpkglist.go.tmp zstdpkglist.go
$(STAMP) $@
@ -3005,6 +3010,7 @@ syscall.lo.dep: $(extra_go_files_syscall)
runtime/internal/sys.lo.dep: $(extra_go_files_runtime_internal_sys)
internal/cpu.lo.dep: $(extra_go_files_internal_cpu)
golang.org/x/sys/cpu.lo.dep: $(extra_go_files_golang_org_x_sys_cpu)
cmd/internal/buildcfg.lo.dep: $(extra_go_files_internal_buildcfg)
internal/goroot.lo.dep: $(extra_go_files_internal_goroot)
go/types.lo.dep: $(extra_go_files_go_types)
cmd/internal/objabi.lo.dep: $(extra_go_files_cmd_internal_objabi)

2
libgo/VERSION
View file

@ -1 +1 @@
go1.16.5
go1.17rc2

6
libgo/check-packages.txt
View file

@ -43,7 +43,10 @@ crypto/des
crypto/dsa
crypto/ecdsa
crypto/ed25519
crypto/ed25519/internal/edwards25519
crypto/ed25519/internal/edwards25519/field
crypto/elliptic
crypto/elliptic/internal/fiat
crypto/hmac
crypto/internal/subtle
crypto/md5
@ -110,6 +113,7 @@ index/suffixarray
internal/cpu
internal/execabs
internal/fmtsort
internal/itoa
internal/poll
internal/profile
internal/reflectlite
@ -139,6 +143,7 @@ net/http/httptest
net/http/httptrace
net/http/httputil
net/http/internal
net/http/internal/ascii
net/http/pprof
net/internal/socktest
net/mail
@ -157,6 +162,7 @@ reflect
regexp
regexp/syntax
runtime
runtime/cgo
runtime/debug
runtime/internal/atomic
runtime/internal/math

2
libgo/configure vendored
View file

@ -2608,7 +2608,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
ac_config_headers="$ac_config_headers config.h"
libtool_VERSION=19:0:0
libtool_VERSION=20:0:0
# Default to --enable-multilib

2
libgo/configure.ac
View file

@ -10,7 +10,7 @@ AC_INIT(package-unused, version-unused,, libgo)
AC_CONFIG_SRCDIR(Makefile.am)
AC_CONFIG_HEADER(config.h)
libtool_VERSION=19:0:0
libtool_VERSION=20:0:0
AC_SUBST(libtool_VERSION)
AM_ENABLE_MULTILIB(, ..)

1
libgo/go/archive/tar/stat_actime1.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build aix || hurd || linux || dragonfly || openbsd || solaris
// +build aix hurd linux dragonfly openbsd solaris
package tar

1
libgo/go/archive/tar/stat_actime2.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build darwin || freebsd || netbsd
// +build darwin freebsd netbsd
package tar

1
libgo/go/archive/tar/stat_unix.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build aix || hurd || linux || darwin || dragonfly || freebsd || openbsd || netbsd || solaris
// +build aix hurd linux darwin dragonfly freebsd openbsd netbsd solaris
package tar

9
libgo/go/archive/tar/tar_test.go
View file

@ -262,16 +262,11 @@ func TestFileInfoHeaderDir(t *testing.T) {
func TestFileInfoHeaderSymlink(t *testing.T) {
testenv.MustHaveSymlink(t)
tmpdir, err := os.MkdirTemp("", "TestFileInfoHeaderSymlink")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
tmpdir := t.TempDir()
link := filepath.Join(tmpdir, "link")
target := tmpdir
err = os.Symlink(target, link)
if err != nil {
if err := os.Symlink(target, link); err != nil {
t.Fatal(err)
}
fi, err := os.Lstat(link)

117
libgo/go/archive/zip/reader.go
View file

@ -52,12 +52,9 @@ type File struct {
FileHeader
zip *Reader
zipr io.ReaderAt
zipsize int64
headerOffset int64
}
func (f *File) hasDataDescriptor() bool {
return f.Flags&0x8 != 0
zip64 bool // zip64 extended information extra field presence
descErr error // error reading the data descriptor during init
}
// OpenReader will open the Zip file specified by name and return a ReadCloser.
@ -120,7 +117,7 @@ func (z *Reader) init(r io.ReaderAt, size int64) error {
// a bad one, and then only report an ErrFormat or UnexpectedEOF if
// the file count modulo 65536 is incorrect.
for {
f := &File{zip: z, zipr: r, zipsize: size}
f := &File{zip: z, zipr: r}
err = readDirectoryHeader(f, buf)
if err == ErrFormat || err == io.ErrUnexpectedEOF {
break
@ -128,6 +125,7 @@ func (z *Reader) init(r io.ReaderAt, size int64) error {
if err != nil {
return err
}
f.readDataDescriptor()
z.File = append(z.File, f)
}
if uint16(len(z.File)) != uint16(end.directoryRecords) { // only compare 16 bits here
@ -188,26 +186,68 @@ func (f *File) Open() (io.ReadCloser, error) {
return nil, ErrAlgorithm
}
var rc io.ReadCloser = dcomp(r)
var desr io.Reader
if f.hasDataDescriptor() {
desr = io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset+size, dataDescriptorLen)
}
rc = &checksumReader{
rc: rc,
hash: crc32.NewIEEE(),
f: f,
desr: desr,
}
return rc, nil
}
// OpenRaw returns a Reader that provides access to the File's contents without
// decompression.
func (f *File) OpenRaw() (io.Reader, error) {
bodyOffset, err := f.findBodyOffset()
if err != nil {
return nil, err
}
r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, int64(f.CompressedSize64))
return r, nil
}
func (f *File) readDataDescriptor() {
if !f.hasDataDescriptor() {
return
}
bodyOffset, err := f.findBodyOffset()
if err != nil {
f.descErr = err
return
}
// In section 4.3.9.2 of the spec: "However ZIP64 format MAY be used
// regardless of the size of a file. When extracting, if the zip64
// extended information extra field is present for the file the
// compressed and uncompressed sizes will be 8 byte values."
//
// Historically, this package has used the compressed and uncompressed
// sizes from the central directory to determine if the package is
// zip64.
//
// For this case we allow either the extra field or sizes to determine
// the data descriptor length.
zip64 := f.zip64 || f.isZip64()
n := int64(dataDescriptorLen)
if zip64 {
n = dataDescriptor64Len
}
size := int64(f.CompressedSize64)
r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset+size, n)
dd, err := readDataDescriptor(r, zip64)
if err != nil {
f.descErr = err
return
}
f.CRC32 = dd.crc32
}
type checksumReader struct {
rc io.ReadCloser
hash hash.Hash32
nread uint64 // number of bytes read so far
f *File
desr io.Reader // if non-nil, where to read the data descriptor
err error // sticky error
err error // sticky error
}
func (r *checksumReader) Stat() (fs.FileInfo, error) {
@ -228,12 +268,12 @@ func (r *checksumReader) Read(b []byte) (n int, err error) {
if r.nread != r.f.UncompressedSize64 {
return 0, io.ErrUnexpectedEOF
}
if r.desr != nil {
if err1 := readDataDescriptor(r.desr, r.f); err1 != nil {
if err1 == io.EOF {
if r.f.hasDataDescriptor() {
if r.f.descErr != nil {
if r.f.descErr == io.EOF {
err = io.ErrUnexpectedEOF
} else {
err = err1
err = r.f.descErr
}
} else if r.hash.Sum32() != r.f.CRC32 {
err = ErrChecksum
@ -344,6 +384,8 @@ parseExtras:
switch fieldTag {
case zip64ExtraID:
f.zip64 = true
// update directory values from the zip64 extra block.
// They should only be consulted if the sizes read earlier
// are maxed out.
@ -443,8 +485,9 @@ parseExtras:
return nil
}
func readDataDescriptor(r io.Reader, f *File) error {
var buf [dataDescriptorLen]byte
func readDataDescriptor(r io.Reader, zip64 bool) (*dataDescriptor, error) {
// Create enough space for the largest possible size
var buf [dataDescriptor64Len]byte
// The spec says: "Although not originally assigned a
// signature, the value 0x08074b50 has commonly been adopted
@ -454,10 +497,9 @@ func readDataDescriptor(r io.Reader, f *File) error {
// descriptors and should account for either case when reading
// ZIP files to ensure compatibility."
//
// dataDescriptorLen includes the size of the signature but
// first read just those 4 bytes to see if it exists.
// First read just those 4 bytes to see if the signature exists.
if _, err := io.ReadFull(r, buf[:4]); err != nil {
return err
return nil, err
}
off := 0
maybeSig := readBuf(buf[:4])
@ -466,21 +508,28 @@ func readDataDescriptor(r io.Reader, f *File) error {
// bytes.
off += 4
}
if _, err := io.ReadFull(r, buf[off:12]); err != nil {
return err
end := dataDescriptorLen - 4
if zip64 {
end = dataDescriptor64Len - 4
}
b := readBuf(buf[:12])
if b.uint32() != f.CRC32 {
return ErrChecksum
if _, err := io.ReadFull(r, buf[off:end]); err != nil {
return nil, err
}
b := readBuf(buf[:end])
out := &dataDescriptor{
crc32: b.uint32(),
}
// The two sizes that follow here can be either 32 bits or 64 bits
// but the spec is not very clear on this and different
// interpretations has been made causing incompatibilities. We
// already have the sizes from the central directory so we can
// just ignore these.
return nil
if zip64 {
out.compressedSize = b.uint64()
out.uncompressedSize = b.uint64()
} else {
out.compressedSize = uint64(b.uint32())
out.uncompressedSize = uint64(b.uint32())
}
return out, nil
}
func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err error) {

167
libgo/go/archive/zip/reader_test.go
View file

@ -499,9 +499,15 @@ func TestReader(t *testing.T) {
func readTestZip(t *testing.T, zt ZipTest) {
var z *Reader
var err error
var raw []byte
if zt.Source != nil {
rat, size := zt.Source()
z, err = NewReader(rat, size)
raw = make([]byte, size)
if _, err := rat.ReadAt(raw, 0); err != nil {
t.Errorf("ReadAt error=%v", err)
return
}
} else {
path := filepath.Join("testdata", zt.Name)
if zt.Obscured {
@ -519,6 +525,12 @@ func readTestZip(t *testing.T, zt ZipTest) {
defer rc.Close()
z = &rc.Reader
}
var err2 error
raw, err2 = os.ReadFile(path)
if err2 != nil {
t.Errorf("ReadFile(%s) error=%v", path, err2)
return
}
}
if err != zt.Error {
t.Errorf("error=%v, want %v", err, zt.Error)
@ -545,7 +557,7 @@ func readTestZip(t *testing.T, zt ZipTest) {
// test read of each file
for i, ft := range zt.File {
readTestFile(t, zt, ft, z.File[i])
readTestFile(t, zt, ft, z.File[i], raw)
}
if t.Failed() {
return
@ -557,7 +569,7 @@ func readTestZip(t *testing.T, zt ZipTest) {
for i := 0; i < 5; i++ {
for j, ft := range zt.File {
go func(j int, ft ZipTestFile) {
readTestFile(t, zt, ft, z.File[j])
readTestFile(t, zt, ft, z.File[j], raw)
done <- true
}(j, ft)
n++
@ -574,7 +586,7 @@ func equalTimeAndZone(t1, t2 time.Time) bool {
return t1.Equal(t2) && name1 == name2 && offset1 == offset2
}
func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File, raw []byte) {
if f.Name != ft.Name {
t.Errorf("name=%q, want %q", f.Name, ft.Name)
}
@ -594,6 +606,31 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
t.Errorf("%v: UncompressedSize=%#x does not match UncompressedSize64=%#x", f.Name, size, f.UncompressedSize64)
}
// Check that OpenRaw returns the correct byte segment
rw, err := f.OpenRaw()
if err != nil {
t.Errorf("%v: OpenRaw error=%v", f.Name, err)
return
}
start, err := f.DataOffset()
if err != nil {
t.Errorf("%v: DataOffset error=%v", f.Name, err)
return
}
got, err := io.ReadAll(rw)
if err != nil {
t.Errorf("%v: OpenRaw ReadAll error=%v", f.Name, err)
return
}
end := uint64(start) + f.CompressedSize64
want := raw[start:end]
if !bytes.Equal(got, want) {
t.Logf("got %q", got)
t.Logf("want %q", want)
t.Errorf("%v: OpenRaw returned unexpected bytes", f.Name)
return
}
r, err := f.Open()
if err != nil {
t.Errorf("%v", err)
@ -776,8 +813,8 @@ func returnRecursiveZip() (r io.ReaderAt, size int64) {
// "archive/zip"
// "bytes"
// "io"
// "io/ioutil"
// "log"
// "os"
// )
//
// type zeros struct{}
@ -1167,6 +1204,128 @@ func TestCVE202127919(t *testing.T) {
}
}
func TestReadDataDescriptor(t *testing.T) {
tests := []struct {
desc string
in []byte
zip64 bool
want *dataDescriptor
wantErr error
}{{
desc: "valid 32 bit with signature",
in: []byte{
0x50, 0x4b, 0x07, 0x08, // signature
0x00, 0x01, 0x02, 0x03, // crc32
0x04, 0x05, 0x06, 0x07, // compressed size
0x08, 0x09, 0x0a, 0x0b, // uncompressed size
},
want: &dataDescriptor{
crc32: 0x03020100,
compressedSize: 0x07060504,
uncompressedSize: 0x0b0a0908,
},
}, {
desc: "valid 32 bit without signature",
in: []byte{
0x00, 0x01, 0x02, 0x03, // crc32
0x04, 0x05, 0x06, 0x07, // compressed size
0x08, 0x09, 0x0a, 0x0b, // uncompressed size
},
want: &dataDescriptor{
crc32: 0x03020100,
compressedSize: 0x07060504,
uncompressedSize: 0x0b0a0908,
},
}, {
desc: "valid 64 bit with signature",
in: []byte{
0x50, 0x4b, 0x07, 0x08, // signature
0x00, 0x01, 0x02, 0x03, // crc32
0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, // compressed size
0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, // uncompressed size
},
zip64: true,
want: &dataDescriptor{
crc32: 0x03020100,
compressedSize: 0x0b0a090807060504,
uncompressedSize: 0x131211100f0e0d0c,
},
}, {
desc: "valid 64 bit without signature",
in: []byte{
0x00, 0x01, 0x02, 0x03, // crc32
0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, // compressed size
0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, // uncompressed size
},
zip64: true,
want: &dataDescriptor{
crc32: 0x03020100,
compressedSize: 0x0b0a090807060504,
uncompressedSize: 0x131211100f0e0d0c,
},
}, {
desc: "invalid 32 bit with signature",
in: []byte{
0x50, 0x4b, 0x07, 0x08, // signature
0x00, 0x01, 0x02, 0x03, // crc32
0x04, 0x05, // unexpected end
},
wantErr: io.ErrUnexpectedEOF,
}, {
desc: "invalid 32 bit without signature",
in: []byte{
0x00, 0x01, 0x02, 0x03, // crc32
0x04, 0x05, // unexpected end
},
wantErr: io.ErrUnexpectedEOF,
}, {
desc: "invalid 64 bit with signature",
in: []byte{
0x50, 0x4b, 0x07, 0x08, // signature
0x00, 0x01, 0x02, 0x03, // crc32
0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, // compressed size
0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, // unexpected end
},
zip64: true,
wantErr: io.ErrUnexpectedEOF,
}, {
desc: "invalid 64 bit without signature",
in: []byte{
0x00, 0x01, 0x02, 0x03, // crc32
0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, // compressed size
0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, // unexpected end
},
zip64: true,
wantErr: io.ErrUnexpectedEOF,
}}
for _, test := range tests {
t.Run(test.desc, func(t *testing.T) {
r := bytes.NewReader(test.in)
desc, err := readDataDescriptor(r, test.zip64)
if err != test.wantErr {
t.Fatalf("got err %v; want nil", err)
}
if test.want == nil {
return
}
if desc == nil {
t.Fatalf("got nil DataDescriptor; want non-nil")
}
if desc.crc32 != test.want.crc32 {
t.Errorf("got CRC32 %#x; want %#x", desc.crc32, test.want.crc32)
}
if desc.compressedSize != test.want.compressedSize {
t.Errorf("got CompressedSize %#x; want %#x", desc.compressedSize, test.want.compressedSize)
}
if desc.uncompressedSize != test.want.uncompressedSize {
t.Errorf("got UncompressedSize %#x; want %#x", desc.uncompressedSize, test.want.uncompressedSize)
}
})
}
}
func TestCVE202133196(t *testing.T) {
// Archive that indicates it has 1 << 128 -1 files,
// this would previously cause a panic due to attempting

22
libgo/go/archive/zip/struct.go
View file

@ -42,7 +42,7 @@ const (
directoryHeaderLen = 46 // + filename + extra + comment
directoryEndLen = 22 // + comment
dataDescriptorLen = 16 // four uint32: descriptor signature, crc32, compressed size, size
dataDescriptor64Len = 24 // descriptor with 8 byte sizes
dataDescriptor64Len = 24 // two uint32: signature, crc32 | two uint64: compressed size, size
directory64LocLen = 20 //
directory64EndLen = 56 // + extra
@ -315,6 +315,10 @@ func (h *FileHeader) isZip64() bool {
return h.CompressedSize64 >= uint32max || h.UncompressedSize64 >= uint32max
}
func (f *FileHeader) hasDataDescriptor() bool {
return f.Flags&0x8 != 0
}
func msdosModeToFileMode(m uint32) (mode fs.FileMode) {
if m&msdosDir != 0 {
mode = fs.ModeDir | 0777
@ -341,11 +345,9 @@ func fileModeToUnixMode(mode fs.FileMode) uint32 {
case fs.ModeSocket:
m = s_IFSOCK
case fs.ModeDevice:
if mode&fs.ModeCharDevice != 0 {
m = s_IFCHR
} else {
m = s_IFBLK
}
m = s_IFBLK
case fs.ModeDevice | fs.ModeCharDevice:
m = s_IFCHR
}
if mode&fs.ModeSetuid != 0 {
m |= s_ISUID
@ -388,3 +390,11 @@ func unixModeToFileMode(m uint32) fs.FileMode {
}
return mode
}
// dataDescriptor holds the data descriptor that optionally follows the file
// contents in the zip file.
type dataDescriptor struct {
crc32 uint32
compressedSize uint64
uncompressedSize uint64
}

135
libgo/go/archive/zip/writer.go
View file

@ -37,6 +37,7 @@ type Writer struct {
type header struct {
*FileHeader
offset uint64
raw bool
}
// NewWriter returns a new Writer writing a zip file to w.
@ -245,22 +246,31 @@ func detectUTF8(s string) (valid, require bool) {
return true, require
}
// prepare performs the bookkeeping operations required at the start of
// CreateHeader and CreateRaw.
func (w *Writer) prepare(fh *FileHeader) error {
if w.last != nil && !w.last.closed {
if err := w.last.close(); err != nil {
return err
}
}
if len(w.dir) > 0 && w.dir[len(w.dir)-1].FileHeader == fh {
// See https://golang.org/issue/11144 confusion.
return errors.New("archive/zip: invalid duplicate FileHeader")
}
return nil
}
// CreateHeader adds a file to the zip archive using the provided FileHeader
// for the file metadata. Writer takes ownership of fh and may mutate
// its fields. The caller must not modify fh after calling CreateHeader.
//
// This returns a Writer to which the file contents should be written.
// The file's contents must be written to the io.Writer before the next
// call to Create, CreateHeader, or Close.
// call to Create, CreateHeader, CreateRaw, or Close.
func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
if w.last != nil && !w.last.closed {
if err := w.last.close(); err != nil {
return nil, err
}
}
if len(w.dir) > 0 && w.dir[len(w.dir)-1].FileHeader == fh {
// See https://golang.org/issue/11144 confusion.
return nil, errors.New("archive/zip: invalid duplicate FileHeader")
if err := w.prepare(fh); err != nil {
return nil, err
}
// The ZIP format has a sad state of affairs regarding character encoding.
@ -365,7 +375,7 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
ow = fw
}
w.dir = append(w.dir, h)
if err := writeHeader(w.cw, fh); err != nil {
if err := writeHeader(w.cw, h); err != nil {
return nil, err
}
// If we're creating a directory, fw is nil.
@ -373,7 +383,7 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
return ow, nil
}
func writeHeader(w io.Writer, h *FileHeader) error {
func writeHeader(w io.Writer, h *header) error {
const maxUint16 = 1<<16 - 1
if len(h.Name) > maxUint16 {
return errLongName
@ -390,9 +400,20 @@ func writeHeader(w io.Writer, h *FileHeader) error {
b.uint16(h.Method)
b.uint16(h.ModifiedTime)
b.uint16(h.ModifiedDate)
b.uint32(0) // since we are writing a data descriptor crc32,
b.uint32(0) // compressed size,
b.uint32(0) // and uncompressed size should be zero
// In raw mode (caller does the compression), the values are either
// written here or in the trailing data descriptor based on the header
// flags.
if h.raw && !h.hasDataDescriptor() {
b.uint32(h.CRC32)
b.uint32(uint32(min64(h.CompressedSize64, uint32max)))
b.uint32(uint32(min64(h.UncompressedSize64, uint32max)))
} else {
// When this package handle the compression, these values are
// always written to the trailing data descriptor.
b.uint32(0) // crc32
b.uint32(0) // compressed size
b.uint32(0) // uncompressed size
}
b.uint16(uint16(len(h.Name)))
b.uint16(uint16(len(h.Extra)))
if _, err := w.Write(buf[:]); err != nil {
@ -405,6 +426,65 @@ func writeHeader(w io.Writer, h *FileHeader) error {
return err
}
func min64(x, y uint64) uint64 {
if x < y {
return x
}
return y
}
// CreateRaw adds a file to the zip archive using the provided FileHeader and
// returns a Writer to which the file contents should be written. The file's
// contents must be written to the io.Writer before the next call to Create,
// CreateHeader, CreateRaw, or Close.
//
// In contrast to CreateHeader, the bytes passed to Writer are not compressed.
func (w *Writer) CreateRaw(fh *FileHeader) (io.Writer, error) {
if err := w.prepare(fh); err != nil {
return nil, err
}
fh.CompressedSize = uint32(min64(fh.CompressedSize64, uint32max))
fh.UncompressedSize = uint32(min64(fh.UncompressedSize64, uint32max))
h := &header{
FileHeader: fh,
offset: uint64(w.cw.count),
raw: true,
}
w.dir = append(w.dir, h)
if err := writeHeader(w.cw, h); err != nil {
return nil, err
}
if strings.HasSuffix(fh.Name, "/") {
w.last = nil
return dirWriter{}, nil
}
fw := &fileWriter{
header: h,
zipw: w.cw,
}
w.last = fw
return fw, nil
}
// Copy copies the file f (obtained from a Reader) into w. It copies the raw
// form directly bypassing decompression, compression, and validation.
func (w *Writer) Copy(f *File) error {
r, err := f.OpenRaw()
if err != nil {
return err
}
fw, err := w.CreateRaw(&f.FileHeader)
if err != nil {
return err
}
_, err = io.Copy(fw, r)
return err
}
// RegisterCompressor registers or overrides a custom compressor for a specific
// method ID. If a compressor for a given method is not found, Writer will
// default to looking up the compressor at the package level.
@ -446,6 +526,9 @@ func (w *fileWriter) Write(p []byte) (int, error) {
if w.closed {
return 0, errors.New("zip: write to closed file")
}
if w.raw {
return w.zipw.Write(p)
}
w.crc32.Write(p)
return w.rawCount.Write(p)
}
@ -455,6 +538,9 @@ func (w *fileWriter) close() error {
return errors.New("zip: file closed twice")
}
w.closed = true
if w.raw {
return w.writeDataDescriptor()
}
if err := w.comp.Close(); err != nil {
return err
}
@ -474,26 +560,33 @@ func (w *fileWriter) close() error {
fh.UncompressedSize = uint32(fh.UncompressedSize64)
}
return w.writeDataDescriptor()
}
func (w *fileWriter) writeDataDescriptor() error {
if !w.hasDataDescriptor() {
return nil
}
// Write data descriptor. This is more complicated than one would
// think, see e.g. comments in zipfile.c:putextended() and
// http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588.
// The approach here is to write 8 byte sizes if needed without
// adding a zip64 extra in the local header (too late anyway).
var buf []byte
if fh.isZip64() {
if w.isZip64() {
buf = make([]byte, dataDescriptor64Len)
} else {
buf = make([]byte, dataDescriptorLen)
}
b := writeBuf(buf)
b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X
b.uint32(fh.CRC32)
if fh.isZip64() {
b.uint64(fh.CompressedSize64)
b.uint64(fh.UncompressedSize64)
b.uint32(w.CRC32)
if w.isZip64() {
b.uint64(w.CompressedSize64)
b.uint64(w.UncompressedSize64)
} else {
b.uint32(fh.CompressedSize)
b.uint32(fh.UncompressedSize)
b.uint32(w.CompressedSize)
b.uint32(w.UncompressedSize)
}
_, err := w.zipw.Write(buf)
return err

185
libgo/go/archive/zip/writer_test.go
View file

@ -6,8 +6,10 @@ package zip
import (
"bytes"
"compress/flate"
"encoding/binary"
"fmt"
"hash/crc32"
"io"
"io/fs"
"math/rand"
@ -57,6 +59,18 @@ var writeTests = []WriteTest{
Method: Deflate,
Mode: 0755 | fs.ModeSymlink,
},
{
Name: "device",
Data: []byte("device file"),
Method: Deflate,
Mode: 0755 | fs.ModeDevice,
},
{
Name: "chardevice",
Data: []byte("char device file"),
Method: Deflate,
Mode: 0755 | fs.ModeDevice | fs.ModeCharDevice,
},
}
func TestWriter(t *testing.T) {
@ -353,6 +367,171 @@ func TestWriterDirAttributes(t *testing.T) {
}
}
func TestWriterCopy(t *testing.T) {
// make a zip file
buf := new(bytes.Buffer)
w := NewWriter(buf)
for _, wt := range writeTests {
testCreate(t, w, &wt)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
// read it back
src, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
if err != nil {
t.Fatal(err)
}
for i, wt := range writeTests {
testReadFile(t, src.File[i], &wt)
}
// make a new zip file copying the old compressed data.
buf2 := new(bytes.Buffer)
dst := NewWriter(buf2)
for _, f := range src.File {
if err := dst.Copy(f); err != nil {
t.Fatal(err)
}
}
if err := dst.Close(); err != nil {
t.Fatal(err)
}
// read the new one back
r, err := NewReader(bytes.NewReader(buf2.Bytes()), int64(buf2.Len()))
if err != nil {
t.Fatal(err)
}
for i, wt := range writeTests {
testReadFile(t, r.File[i], &wt)
}
}
func TestWriterCreateRaw(t *testing.T) {
files := []struct {
name string
content []byte
method uint16
flags uint16
crc32 uint32
uncompressedSize uint64
compressedSize uint64
}{
{
name: "small store w desc",
content: []byte("gophers"),
method: Store,
flags: 0x8,
},
{
name: "small deflate wo desc",
content: bytes.Repeat([]byte("abcdefg"), 2048),
method: Deflate,
},
}
// write a zip file
archive := new(bytes.Buffer)
w := NewWriter(archive)
for i := range files {
f := &files[i]
f.crc32 = crc32.ChecksumIEEE(f.content)
size := uint64(len(f.content))
f.uncompressedSize = size
f.compressedSize = size
var compressedContent []byte
if f.method == Deflate {
var buf bytes.Buffer
w, err := flate.NewWriter(&buf, flate.BestSpeed)
if err != nil {
t.Fatalf("flate.NewWriter err = %v", err)
}
_, err = w.Write(f.content)
if err != nil {
t.Fatalf("flate Write err = %v", err)
}
err = w.Close()
if err != nil {
t.Fatalf("flate Writer.Close err = %v", err)
}
compressedContent = buf.Bytes()
f.compressedSize = uint64(len(compressedContent))
}
h := &FileHeader{
Name: f.name,
Method: f.method,
Flags: f.flags,
CRC32: f.crc32,
CompressedSize64: f.compressedSize,
UncompressedSize64: f.uncompressedSize,
}
w, err := w.CreateRaw(h)
if err != nil {
t.Fatal(err)
}
if compressedContent != nil {
_, err = w.Write(compressedContent)
} else {
_, err = w.Write(f.content)
}
if err != nil {
t.Fatalf("%s Write got %v; want nil", f.name, err)
}
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
// read it back
r, err := NewReader(bytes.NewReader(archive.Bytes()), int64(archive.Len()))
if err != nil {
t.Fatal(err)
}
for i, want := range files {
got := r.File[i]
if got.Name != want.name {
t.Errorf("got Name %s; want %s", got.Name, want.name)
}
if got.Method != want.method {
t.Errorf("%s: got Method %#x; want %#x", want.name, got.Method, want.method)
}
if got.Flags != want.flags {
t.Errorf("%s: got Flags %#x; want %#x", want.name, got.Flags, want.flags)
}
if got.CRC32 != want.crc32 {
t.Errorf("%s: got CRC32 %#x; want %#x", want.name, got.CRC32, want.crc32)
}
if got.CompressedSize64 != want.compressedSize {
t.Errorf("%s: got CompressedSize64 %d; want %d", want.name, got.CompressedSize64, want.compressedSize)
}
if got.UncompressedSize64 != want.uncompressedSize {
t.Errorf("%s: got UncompressedSize64 %d; want %d", want.name, got.UncompressedSize64, want.uncompressedSize)
}
r, err := got.Open()
if err != nil {
t.Errorf("%s: Open err = %v", got.Name, err)
continue
}
buf, err := io.ReadAll(r)
if err != nil {
t.Errorf("%s: ReadAll err = %v", got.Name, err)
continue
}
if !bytes.Equal(buf, want.content) {
t.Errorf("%v: ReadAll returned unexpected bytes", got.Name)
}
}
}
func testCreate(t *testing.T, w *Writer, wt *WriteTest) {
header := &FileHeader{
Name: wt.Name,
@ -378,15 +557,15 @@ func testReadFile(t *testing.T, f *File, wt *WriteTest) {
testFileMode(t, f, wt.Mode)
rc, err := f.Open()
if err != nil {
t.Fatal("opening:", err)
t.Fatalf("opening %s: %v", f.Name, err)
}
b, err := io.ReadAll(rc)
if err != nil {
t.Fatal("reading:", err)
t.Fatalf("reading %s: %v", f.Name, err)
}
err = rc.Close()
if err != nil {
t.Fatal("closing:", err)
t.Fatalf("closing %s: %v", f.Name, err)
}
if !bytes.Equal(b, wt.Data) {
t.Errorf("File contents %q, want %q", b, wt.Data)

3
libgo/go/bufio/bufio.go
View file

@ -670,7 +670,8 @@ func (b *Writer) WriteByte(c byte) error {
// WriteRune writes a single Unicode code point, returning
// the number of bytes written and any error.
func (b *Writer) WriteRune(r rune) (size int, err error) {
if r < utf8.RuneSelf {
// Compare as uint32 to correctly handle negative runes.
if uint32(r) < utf8.RuneSelf {
err = b.WriteByte(byte(r))
if err != nil {
return 0, err

14
libgo/go/bufio/bufio_test.go
View file

@ -534,6 +534,20 @@ func TestReadWriteRune(t *testing.T) {
}
}
func TestWriteInvalidRune(t *testing.T) {
// Invalid runes, including negative ones, should be written as the
// replacement character.
for _, r := range []rune{-1, utf8.MaxRune + 1} {
var buf bytes.Buffer
w := NewWriter(&buf)
w.WriteRune(r)
w.Flush()
if s := buf.String(); s != "\uFFFD" {
t.Errorf("WriteRune(%d) wrote %q, not replacement character", r, s)
}
}
}
func TestReadStringAllocs(t *testing.T) {
r := strings.NewReader(" foo foo 42 42 42 42 42 42 42 42 4.2 4.2 4.2 4.2\n")
buf := NewReader(r)

3
libgo/go/bufio/scan.go
View file

@ -48,7 +48,8 @@ type Scanner struct {
// and the next token to return to the user, if any, plus an error, if any.
//
// Scanning stops if the function returns an error, in which case some of
// the input may be discarded.
// the input may be discarded. If that error is ErrFinalToken, scanning
// stops with no error.
//
// Otherwise, the Scanner advances the input. If the token is not nil,
// the Scanner returns it to the user. If the token is nil, the

1
libgo/go/bytes/boundary_test.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
//go:build linux
// +build linux
package bytes_test

3
libgo/go/bytes/buffer.go
View file

@ -275,7 +275,8 @@ func (b *Buffer) WriteByte(c byte) error {
// included to match bufio.Writer's WriteRune. The buffer is grown as needed;
// if it becomes too large, WriteRune will panic with ErrTooLarge.
func (b *Buffer) WriteRune(r rune) (n int, err error) {
if r < utf8.RuneSelf {
// Compare as uint32 to correctly handle negative runes.
if uint32(r) < utf8.RuneSelf {
b.WriteByte(byte(r))
return 1, nil
}

11
libgo/go/bytes/buffer_test.go
View file

@ -6,6 +6,7 @@ package bytes_test
import (
. "bytes"
"fmt"
"io"
"math/rand"
"testing"
@ -387,6 +388,16 @@ func TestRuneIO(t *testing.T) {
}
}
func TestWriteInvalidRune(t *testing.T) {
// Invalid runes, including negative ones, should be written as
// utf8.RuneError.
for _, r := range []rune{-1, utf8.MaxRune + 1} {
var buf Buffer
buf.WriteRune(r)
check(t, fmt.Sprintf("TestWriteInvalidRune (%d)", r), &buf, "\uFFFD")
}
}
func TestNext(t *testing.T) {
b := []byte{0, 1, 2, 3, 4}
tmp := make([]byte, 5)

3
libgo/go/cmd/cgo/doc.go
View file

@ -387,6 +387,9 @@ and of course there is nothing stopping the C code from doing anything
it likes. However, programs that break these rules are likely to fail
in unexpected and unpredictable ways.
The runtime/cgo.Handle type can be used to safely pass Go values
between Go and C. See the runtime/cgo package documentation for details.
Note: the current implementation has a bug. While Go code is permitted
to write nil or a C pointer (but not a Go pointer) to C memory, the
current implementation may sometimes cause a runtime error if the

48
libgo/go/cmd/cgo/gcc.go
View file

@ -927,7 +927,7 @@ func (p *Package) rewriteCall(f *File, call *Call) (string, bool) {
var sbCheck bytes.Buffer
for i, param := range params {
origArg := args[i]
arg, nu := p.mangle(f, &args[i])
arg, nu := p.mangle(f, &args[i], true)
if nu {
needsUnsafe = true
}
@ -970,7 +970,7 @@ func (p *Package) rewriteCall(f *File, call *Call) (string, bool) {
sb.WriteString("return ")
}
m, nu := p.mangle(f, &call.Call.Fun)
m, nu := p.mangle(f, &call.Call.Fun, false)
if nu {
needsUnsafe = true
}
@ -1104,7 +1104,8 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
// rewriting calls when it finds them.
// It removes the corresponding references in f.Ref and f.Calls, so that we
// don't try to do the replacement again in rewriteRef or rewriteCall.
func (p *Package) mangle(f *File, arg *ast.Expr) (ast.Expr, bool) {
// If addPosition is true, add position info to the idents of C names in arg.
func (p *Package) mangle(f *File, arg *ast.Expr, addPosition bool) (ast.Expr, bool) {
needsUnsafe := false
f.walk(arg, ctxExpr, func(f *File, arg interface{}, context astContext) {
px, ok := arg.(*ast.Expr)
@ -1119,7 +1120,7 @@ func (p *Package) mangle(f *File, arg *ast.Expr) (ast.Expr, bool) {
for _, r := range f.Ref {
if r.Expr == px {
*px = p.rewriteName(f, r)
*px = p.rewriteName(f, r, addPosition)
r.Done = true
break
}
@ -1379,7 +1380,7 @@ func (p *Package) rewriteRef(f *File) {
}
}
expr := p.rewriteName(f, r)
expr := p.rewriteName(f, r, false)
if *godefs {
// Substitute definition for mangled type name.
@ -1442,8 +1443,23 @@ func (p *Package) rewriteRef(f *File) {
}
// rewriteName returns the expression used to rewrite a reference.
func (p *Package) rewriteName(f *File, r *Ref) ast.Expr {
var expr ast.Expr = ast.NewIdent(r.Name.Mangle) // default
// If addPosition is true, add position info in the ident name.
func (p *Package) rewriteName(f *File, r *Ref, addPosition bool) ast.Expr {
getNewIdent := ast.NewIdent
if addPosition {
getNewIdent = func(newName string) *ast.Ident {
mangledIdent := ast.NewIdent(newName)
if len(newName) == len(r.Name.Go) {
return mangledIdent
}
p := fset.Position((*r.Expr).End())
if p.Column == 0 {
return mangledIdent
}
return ast.NewIdent(fmt.Sprintf("%s /*line :%d:%d*/", newName, p.Line, p.Column))
}
}
var expr ast.Expr = getNewIdent(r.Name.Mangle) // default
switch r.Context {
case ctxCall, ctxCall2:
if r.Name.Kind != "func" {
@ -1471,7 +1487,7 @@ func (p *Package) rewriteName(f *File, r *Ref) ast.Expr {
n.Mangle = "_C2func_" + n.Go
f.Name["2"+r.Name.Go] = n
}
expr = ast.NewIdent(n.Mangle)
expr = getNewIdent(n.Mangle)
r.Name = n
break
}
@ -1502,7 +1518,7 @@ func (p *Package) rewriteName(f *File, r *Ref) ast.Expr {
// issue 7757.
expr = &ast.CallExpr{
Fun: &ast.Ident{NamePos: (*r.Expr).Pos(), Name: "_Cgo_ptr"},
Args: []ast.Expr{ast.NewIdent(name.Mangle)},
Args: []ast.Expr{getNewIdent(name.Mangle)},
}
case "type":
// Okay - might be new(T)
@ -1584,9 +1600,17 @@ func (p *Package) gccMachine() []string {
case "s390x":
return []string{"-m64"}
case "mips64", "mips64le":
return []string{"-mabi=64"}
if gomips64 == "hardfloat" {
return []string{"-mabi=64", "-mhard-float"}
} else if gomips64 == "softfloat" {
return []string{"-mabi=64", "-msoft-float"}
}
case "mips", "mipsle":
return []string{"-mabi=32"}
if gomips == "hardfloat" {
return []string{"-mabi=32", "-mfp32", "-mhard-float", "-mno-odd-spreg"}
} else if gomips == "softfloat" {
return []string{"-mabi=32", "-msoft-float"}
}
case "ppc64":
if goos == "aix" {
return []string{"-maix64"}
@ -1639,6 +1663,8 @@ func (p *Package) gccCmd() []string {
if goos == "aix" {
c = append(c, "-mcmodel=large")
}
// disable LTO so we get an object whose symbols we can read
c = append(c, "-fno-lto")
c = append(c, "-") //read input from standard input
return c
}

17
libgo/go/cmd/cgo/main.go
View file

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Cgo; see gmp.go for an overview.
// Cgo; see doc.go for an overview.
// TODO(rsc):
// Emit correct line number annotations.
@ -17,9 +17,11 @@ import (
"go/ast"
"go/printer"
"go/token"
"internal/buildcfg"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"reflect"
"runtime"
@ -249,7 +251,7 @@ var importRuntimeCgo = flag.Bool("import_runtime_cgo", true, "import runtime/cgo
var importSyscall = flag.Bool("import_syscall", true, "import syscall in generated code")
var trimpath = flag.String("trimpath", "", "applies supplied rewrites or trims prefixes to recorded source file paths")
var goarch, goos string
var goarch, goos, gomips, gomips64 string
func main() {
objabi.AddVersionFlag() // -V
@ -306,6 +308,14 @@ func main() {
p := newPackage(args[:i])
// We need a C compiler to be available. Check this.
gccName := p.gccBaseCmd()[0]
_, err := exec.LookPath(gccName)
if err != nil {
fatalf("C compiler %q not found: %v", gccName, err)
os.Exit(2)
}
// Record CGO_LDFLAGS from the environment for external linking.
if ldflags := os.Getenv("CGO_LDFLAGS"); ldflags != "" {
args, err := splitQuoted(ldflags)
@ -409,6 +419,9 @@ func newPackage(args []string) *Package {
if s := os.Getenv("GOOS"); s != "" {
goos = s
}
buildcfg.Check()
gomips = buildcfg.GOMIPS
gomips64 = buildcfg.GOMIPS64
ptrSize := ptrSizeMap[goarch]
if ptrSize == 0 {
fatalf("unknown ptrSize for $GOARCH %q", goarch)

55
libgo/go/cmd/cgo/out.go
View file

@ -173,8 +173,18 @@ func (p *Package) writeDefs() {
if *gccgo {
fmt.Fprintf(fc, "extern byte *%s;\n", n.C)
} else {
fmt.Fprintf(fm, "extern char %s[];\n", n.C)
fmt.Fprintf(fm, "void *_cgohack_%s = %s;\n\n", n.C, n.C)
// Force a reference to all symbols so that
// the external linker will add DT_NEEDED
// entries as needed on ELF systems.
// Treat function variables differently
// to avoid type confict errors from LTO
// (Link Time Optimization).
if n.Kind == "fpvar" {
fmt.Fprintf(fm, "extern void %s();\n", n.C)
} else {
fmt.Fprintf(fm, "extern char %s[];\n", n.C)
fmt.Fprintf(fm, "void *_cgohack_%s = %s;\n\n", n.C, n.C)
}
fmt.Fprintf(fgo2, "//go:linkname __cgo_%s %s\n", n.C, n.C)
fmt.Fprintf(fgo2, "//go:cgo_import_static %s\n", n.C)
fmt.Fprintf(fgo2, "var __cgo_%s byte\n", n.C)
@ -1026,14 +1036,28 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
}
fmt.Fprintf(fgcc, "}\n")
// In internal linking mode, the Go linker sees both
// the C wrapper written above and the Go wrapper it
// references. Hence, export the C wrapper (e.g., for
// if we're building a shared object). The Go linker
// will resolve the C wrapper's reference to the Go
// wrapper without a separate export.
fmt.Fprintf(fgo2, "//go:cgo_export_dynamic %s\n", exp.ExpName)
// cgo_export_static refers to a symbol by its linker
// name, so set the linker name of the Go wrapper.
fmt.Fprintf(fgo2, "//go:linkname _cgoexp%s_%s _cgoexp%s_%s\n", cPrefix, exp.ExpName, cPrefix, exp.ExpName)
// In external linking mode, the Go linker sees the Go
// wrapper, but not the C wrapper. For this case,
// export the Go wrapper so the host linker can
// resolve the reference from the C wrapper to the Go
// wrapper.
fmt.Fprintf(fgo2, "//go:cgo_export_static _cgoexp%s_%s\n", cPrefix, exp.ExpName)
// Build the wrapper function compiled by cmd/compile.
// This unpacks the argument struct above and calls the Go function.
fmt.Fprintf(fgo2, "//go:cgo_export_dynamic %s\n", exp.ExpName)
fmt.Fprintf(fgo2, "//go:linkname _cgoexp%s_%s _cgoexp%s_%s\n", cPrefix, exp.ExpName, cPrefix, exp.ExpName)
fmt.Fprintf(fgo2, "//go:cgo_export_static _cgoexp%s_%s\n", cPrefix, exp.ExpName)
fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a *%s) {\n", cPrefix, exp.ExpName, gotype)
fmt.Fprintf(fm, "int _cgoexp%s_%s;\n", cPrefix, exp.ExpName)
fmt.Fprintf(fm, "void _cgoexp%s_%s(void* p){}\n", cPrefix, exp.ExpName)
if gccResult != "void" {
// Write results back to frame.
@ -1722,8 +1746,12 @@ typedef struct __go_open_array {
struct __go_string __go_byte_array_to_string(const void* p, intgo len);
struct __go_open_array __go_string_to_byte_array (struct __go_string str);
extern void runtime_throw(const char *);
const char *_cgoPREFIX_Cfunc_CString(struct __go_string s) {
char *p = malloc(s.__length+1);
if(p == NULL)
runtime_throw("runtime: C malloc failed");
memmove(p, s.__data, s.__length);
p[s.__length] = 0;
return p;
@ -1731,6 +1759,8 @@ const char *_cgoPREFIX_Cfunc_CString(struct __go_string s) {
void *_cgoPREFIX_Cfunc_CBytes(struct __go_open_array b) {
char *p = malloc(b.__count);
if(p == NULL)
runtime_throw("runtime: C malloc failed");
memmove(p, b.__values, b.__count);
return p;
}
@ -1749,14 +1779,13 @@ Slice _cgoPREFIX_Cfunc_GoBytes(char *p, int32_t n) {
return __go_string_to_byte_array(s);
}
extern void runtime_throw(const char *);
void *_cgoPREFIX_Cfunc__CMalloc(size_t n) {
void *p = malloc(n);
if(p == NULL && n == 0)
p = malloc(1);
if(p == NULL)
runtime_throw("runtime: C malloc failed");
return p;
void *p = malloc(n);
if(p == NULL && n == 0)
p = malloc(1);
if(p == NULL)
runtime_throw("runtime: C malloc failed");
return p;
}
struct __go_type_descriptor;

17
libgo/go/cmd/go.mod
View file

@ -1,12 +1,15 @@
module cmd
go 1.16
go 1.17
require (
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2
golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897
golang.org/x/mod v0.4.2-0.20210325185522-dbbbf8a3c6ea
golang.org/x/sys v0.0.0-20201204225414-ed752295db88 // indirect
golang.org/x/tools v0.0.0-20210107193943-4ed967dd8eff
github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 // indirect
golang.org/x/arch v0.0.0-20210502124803-cbf565b21d1e
golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e // indirect
golang.org/x/mod v0.4.3-0.20210608190319-0f08993efd8a
golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744 // indirect
golang.org/x/term v0.0.0-20210503060354-a79de5458b56
golang.org/x/tools v0.1.2-0.20210519160823-49064d2332f9
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
)

151
libgo/go/cmd/go/alldocs.go
View file

@ -111,7 +111,7 @@
// -p n
// the number of programs, such as build commands or
// test binaries, that can be run in parallel.
// The default is the number of CPUs available.
// The default is GOMAXPROCS, normally the number of CPUs available.
// -race
// enable data race detection.
// Supported only on linux/amd64, freebsd/amd64, darwin/amd64, windows/amd64,
@ -174,8 +174,8 @@
// a build will run as if the disk file path exists with the contents
// given by the backing file paths, or as if the disk file path does not
// exist if its backing file path is empty. Support for the -overlay flag
// has some limitations:importantly, cgo files included from outside the
// include path must be in the same directory as the Go package they are
// has some limitations: importantly, cgo files included from outside the
// include path must be in the same directory as the Go package they are
// included from, and overlays will not appear when binaries and tests are
// run through go run and go test respectively.
// -pkgdir dir
@ -198,6 +198,8 @@
// a program to use to invoke toolchain programs like vet and asm.
// For example, instead of running asm, the go command will run
// 'cmd args /path/to/asm <arguments for asm>'.
// The TOOLEXEC_IMPORTPATH environment variable will be set,
// matching 'go list -f {{.ImportPath}}' for the package being built.
//
// The -asmflags, -gccgoflags, -gcflags, and -ldflags flags accept a
// space-separated list of arguments to pass to an underlying tool
@ -291,7 +293,7 @@
//
// Usage:
//
// go doc [-u] [-c] [package|[package.]symbol[.methodOrField]]
// go doc [doc flags] [package|[package.]symbol[.methodOrField]]
//
// Doc prints the documentation comments associated with the item identified by its
// arguments (a package, const, func, type, var, method, or struct field)
@ -596,7 +598,7 @@
//
// Usage:
//
// go get [-d] [-t] [-u] [-v] [-insecure] [build flags] [packages]
// go get [-d] [-t] [-u] [-v] [build flags] [packages]
//
// Get resolves its command-line arguments to packages at specific module versions,
// updates go.mod to require those versions, downloads source code into the
@ -641,14 +643,6 @@
// When the -t and -u flags are used together, get will update
// test dependencies as well.
//
// The -insecure flag permits fetching from repositories and resolving
// custom domains using insecure schemes such as HTTP, and also bypassess
// module sum validation using the checksum database. Use with caution.
// This flag is deprecated and will be removed in a future version of go.
// To permit the use of insecure schemes, use the GOINSECURE environment
// variable instead. To bypass module sum validation, use GOPRIVATE or
// GONOSUMDB. See 'go help environment' for details.
//
// The -d flag instructs get not to build or install packages. get will only
// update go.mod and download source code needed to build packages.
//
@ -849,6 +843,7 @@
// UseAllFiles bool // use files regardless of +build lines, file names
// Compiler string // compiler to assume when computing target paths
// BuildTags []string // build constraints to match in +build lines
// ToolTags []string // toolchain-specific build constraints
// ReleaseTags []string // releases the current release is compatible with
// InstallSuffix string // suffix to use in the name of the install dir
// }
@ -1083,7 +1078,7 @@
//
// Usage:
//
// go mod edit [editing flags] [go.mod]
// go mod edit [editing flags] [-fmt|-print|-json] [go.mod]
//
// Edit provides a command-line interface for editing go.mod,
// for use primarily by tools or scripts. It reads only go.mod;
@ -1142,12 +1137,12 @@
// writing it back to go.mod. The JSON output corresponds to these Go types:
//
// type Module struct {
// Path string
// Path string
// Version string
// }
//
// type GoMod struct {
// Module Module
// Module ModPath
// Go string
// Require []Require
// Exclude []Module
@ -1155,6 +1150,11 @@
// Retract []Retract
// }
//
// type ModPath struct {
// Path string
// Deprecated string
// }
//
// type Require struct {
// Path string
// Version string
@ -1186,13 +1186,17 @@
//
// Usage:
//
// go mod graph
// go mod graph [-go=version]
//
// Graph prints the module requirement graph (with replacements applied)
// in text form. Each line in the output has two space-separated fields: a module
// and one of its requirements. Each module is identified as a string of the form
// path@version, except for the main module, which has no @version suffix.
//
// The -go flag causes graph to report the module graph as loaded by the
// given Go version, instead of the version indicated by the 'go' directive
// in the go.mod file.
//
// See https://golang.org/ref/mod#go-mod-graph for more about 'go mod graph'.
//
//
@ -1200,7 +1204,7 @@
//
// Usage:
//
// go mod init [module]
// go mod init [module-path]
//
// Init initializes and writes a new go.mod file in the current directory, in
// effect creating a new module rooted at the current directory. The go.mod file
@ -1221,7 +1225,7 @@
//
// Usage:
//
// go mod tidy [-e] [-v]
// go mod tidy [-e] [-v] [-go=version] [-compat=version]
//
// Tidy makes sure go.mod matches the source code in the module.
// It adds any missing modules necessary to build the current module's
@ -1235,6 +1239,20 @@
// The -e flag causes tidy to attempt to proceed despite errors
// encountered while loading packages.
//
// The -go flag causes tidy to update the 'go' directive in the go.mod
// file to the given version, which may change which module dependencies
// are retained as explicit requirements in the go.mod file.
// (Go versions 1.17 and higher retain more requirements in order to
// support lazy module loading.)
//
// The -compat flag preserves any additional checksums needed for the
// 'go' command from the indicated major Go release to successfully load
// the module graph, and causes tidy to error out if that version of the
// 'go' command would load any imported package from a different module
// version. By default, tidy acts as if the -compat flag were set to the
// version prior to the one indicated by the 'go' directive in the go.mod
// file.
//
// See https://golang.org/ref/mod#go-mod-tidy for more about 'go mod tidy'.
//
//
@ -1318,10 +1336,21 @@
// go run [build flags] [-exec xprog] package [arguments...]
//
// Run compiles and runs the named main Go package.
// Typically the package is specified as a list of .go source files from a single directory,
// but it may also be an import path, file system path, or pattern
// Typically the package is specified as a list of .go source files from a single
// directory, but it may also be an import path, file system path, or pattern
// matching a single known package, as in 'go run .' or 'go run my/cmd'.
//
// If the package argument has a version suffix (like @latest or @v1.0.0),
// "go run" builds the program in module-aware mode, ignoring the go.mod file in
// the current directory or any parent directory, if there is one. This is useful
// for running programs without affecting the dependencies of the main module.
//
// If the package argument doesn't have a version suffix, "go run" may run in
// module-aware mode or GOPATH mode, depending on the GO111MODULE environment
// variable and the presence of a go.mod file. See 'go help modules' for details.
// If module-aware mode is enabled, "go run" runs in the context of the main
// module.
//
// By default, 'go run' runs the compiled binary directly: 'a.out arguments...'.
// If the -exec flag is given, 'go run' invokes the binary using xprog:
// 'xprog a.out arguments...'.
@ -1416,8 +1445,8 @@
//
// The rule for a match in the cache is that the run involves the same
// test binary and the flags on the command line come entirely from a
// restricted set of 'cacheable' test flags, defined as -cpu, -list,
// -parallel, -run, -short, and -v. If a run of go test has any test
// restricted set of 'cacheable' test flags, defined as -benchtime, -cpu,
// -list, -parallel, -run, -short, and -v. If a run of go test has any test
// or non-test flags outside this set, the result is not cached. To
// disable test caching, use any test flag or argument other than the
// cacheable flags. The idiomatic way to disable test caching explicitly
@ -1543,7 +1572,7 @@
//
// A build constraint, also known as a build tag, is a line comment that begins
//
// // +build
// //go:build
//
// that lists the conditions under which a file should be included in the package.
// Constraints may appear in any kind of source file (not just Go), but
@ -1551,30 +1580,20 @@
// only by blank lines and other line comments. These rules mean that in Go
// files a build constraint must appear before the package clause.
//
// To distinguish build constraints from package documentation, a series of
// build constraints must be followed by a blank line.
// To distinguish build constraints from package documentation,
// a build constraint should be followed by a blank line.
//
// A build constraint is evaluated as the OR of space-separated options.
// Each option evaluates as the AND of its comma-separated terms.
// Each term consists of letters, digits, underscores, and dots.
// A term may be negated with a preceding !.
// For example, the build constraint:
// A build constraint is evaluated as an expression containing options
// combined by ||, &&, and ! operators and parentheses. Operators have
// the same meaning as in Go.
//
// // +build linux,386 darwin,!cgo
// For example, the following build constraint constrains a file to
// build when the "linux" and "386" constraints are satisfied, or when
// "darwin" is satisfied and "cgo" is not:
//
// corresponds to the boolean formula:
// //go:build (linux && 386) || (darwin && !cgo)
//
// (linux AND 386) OR (darwin AND (NOT cgo))
//
// A file may have multiple build constraints. The overall constraint is the AND
// of the individual constraints. That is, the build constraints:
//
// // +build linux darwin
// // +build amd64
//
// corresponds to the boolean formula:
//
// (linux OR darwin) AND amd64
// It is an error for a file to have more than one //go:build line.
//
// During a particular build, the following words are satisfied:
//
@ -1612,24 +1631,28 @@
//
// To keep a file from being considered for the build:
//
// // +build ignore
// //go:build ignore
//
// (any other unsatisfied word will work as well, but "ignore" is conventional.)
//
// To build a file only when using cgo, and only on Linux and OS X:
//
// // +build linux,cgo darwin,cgo
// //go:build cgo && (linux || darwin)
//
// Such a file is usually paired with another file implementing the
// default functionality for other systems, which in this case would
// carry the constraint:
//
// // +build !linux,!darwin !cgo
// //go:build !(cgo && (linux || darwin))
//
// Naming a file dns_windows.go will cause it to be included only when
// building the package for Windows; similarly, math_386.s will be included
// only when building the package for 32-bit x86.
//
// Go versions 1.16 and earlier used a different syntax for build constraints,
// with a "// +build" prefix. The gofmt command will add an equivalent //go:build
// constraint when encountering the older syntax.
//
//
// Build modes
//
@ -1787,9 +1810,8 @@
// Comma-separated list of glob patterns (in the syntax of Go's path.Match)
// of module path prefixes that should always be fetched in an insecure
// manner. Only applies to dependencies that are being fetched directly.
// Unlike the -insecure flag on 'go get', GOINSECURE does not disable
// checksum database validation. GOPRIVATE or GONOSUMDB may be used
// to achieve that.
// GOINSECURE does not disable checksum database validation. GOPRIVATE or
// GONOSUMDB may be used to achieve that.
// GOOS
// The operating system for which to compile code.
// Examples are linux, darwin, windows, netbsd.
@ -1869,6 +1891,9 @@
// GOMIPS64
// For GOARCH=mips64{,le}, whether to use floating point instructions.
// Valid values are hardfloat (default), softfloat.
// GOPPC64
// For GOARCH=ppc64{,le}, the target ISA (Instruction Set Architecture).
// Valid values are power8 (default), power9.
// GOWASM
// For GOARCH=wasm, comma-separated list of experimental WebAssembly features to use.
// Valid values are satconv, signext.
@ -1878,6 +1903,12 @@
// GCCGOTOOLDIR
// If set, where to find gccgo tools, such as cgo.
// The default is based on how gccgo was configured.
// GOEXPERIMENT
// Comma-separated list of toolchain experiments to enable or disable.
// The list of available experiments may change arbitrarily over time.
// See src/internal/goexperiment/flags.go for currently valid values.
// Warning: This variable is provided for the development and testing
// of the Go toolchain itself. Use beyond that purpose is unsupported.
// GOROOT_FINAL
// The root of the installed Go tree, when it is
// installed in a location other than where it is built.
@ -1961,7 +1992,7 @@
// The go.mod file format is described in detail at
// https://golang.org/ref/mod#go-mod-file.
//
// To create a new go.mod file, use 'go help init'. For details see
// To create a new go.mod file, use 'go mod init'. For details see
// 'go help mod init' or https://golang.org/ref/mod#go-mod-init.
//
// To add missing module requirements or remove unneeded requirements,
@ -2139,7 +2170,7 @@
// This help text, accessible as 'go help gopath-get' even in module-aware mode,
// describes 'go get' as it operates in legacy GOPATH mode.
//
// Usage: go get [-d] [-f] [-t] [-u] [-v] [-fix] [-insecure] [build flags] [packages]
// Usage: go get [-d] [-f] [-t] [-u] [-v] [-fix] [build flags] [packages]
//
// Get downloads the packages named by the import paths, along with their
// dependencies. It then installs the named packages, like 'go install'.
@ -2155,13 +2186,6 @@
// The -fix flag instructs get to run the fix tool on the downloaded packages
// before resolving dependencies or building the code.
//
// The -insecure flag permits fetching from repositories and resolving
// custom domains using insecure schemes such as HTTP. Use with caution.
// This flag is deprecated and will be removed in a future version of go.
// The GOINSECURE environment variable should be used instead, since it
// provides control over which packages may be retrieved using an insecure
// scheme. See 'go help environment' for details.
//
// The -t flag instructs get to also download the packages required to build
// the tests for the specified packages.
//
@ -2346,7 +2370,7 @@
// will result in the following requests:
//
// https://example.org/pkg/foo?go-get=1 (preferred)
// http://example.org/pkg/foo?go-get=1 (fallback, only with -insecure)
// http://example.org/pkg/foo?go-get=1 (fallback, only with use of correctly set GOINSECURE)
//
// If that page contains the meta tag
//
@ -2664,6 +2688,13 @@
// the Go tree can run a sanity check but not spend time running
// exhaustive tests.
//
// -shuffle off,on,N
// Randomize the execution order of tests and benchmarks.
// It is off by default. If -shuffle is set to on, then it will seed
// the randomizer using the system clock. If -shuffle is set to an
// integer N, then N will be used as the seed value. In both cases,
// the seed will be reported for reproducibility.
//
// -timeout d
// If a test binary runs longer than duration d, panic.
// If d is 0, the timeout is disabled.

1
libgo/go/cmd/go/go11.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build go1.1
// +build go1.1
package main

68
libgo/go/cmd/go/go_test.go
View file

@ -72,7 +72,6 @@ func tooSlow(t *testing.T) {
// (temp) directory.
var testGOROOT string
var testCC string
var testGOCACHE string
var testGo string
@ -179,13 +178,6 @@ func TestMain(m *testing.M) {
os.Exit(2)
}
out, err = exec.Command(gotool, "env", "CC").CombinedOutput()
if err != nil {
fmt.Fprintf(os.Stderr, "could not find testing CC: %v\n%s", err, out)
os.Exit(2)
}
testCC = strings.TrimSpace(string(out))
cmd := exec.Command(testGo, "env", "CGO_ENABLED")
cmd.Stderr = new(strings.Builder)
if out, err := cmd.Output(); err != nil {
@ -811,8 +803,10 @@ func TestNewReleaseRebuildsStalePackagesInGOPATH(t *testing.T) {
// so that we can change files.
for _, copydir := range []string{
"src/runtime",
"src/internal/abi",
"src/internal/bytealg",
"src/internal/cpu",
"src/internal/goexperiment",
"src/math/bits",
"src/unsafe",
filepath.Join("pkg", runtime.GOOS+"_"+runtime.GOARCH),
@ -2183,7 +2177,7 @@ func testBuildmodePIE(t *testing.T, useCgo, setBuildmodeToPIE bool) {
// See https://sourceware.org/bugzilla/show_bug.cgi?id=19011
section := f.Section(".edata")
if section == nil {
t.Fatalf(".edata section is not present")
t.Skip(".edata section is not present")
}
// TODO: deduplicate this struct from cmd/link/internal/ld/pe.go
type IMAGE_EXPORT_DIRECTORY struct {
@ -2830,3 +2824,59 @@ func TestCoverpkgTestOnly(t *testing.T) {
tg.grepStderrNot("no packages being tested depend on matches", "bad match message")
tg.grepStdout("coverage: 100", "no coverage")
}
// Regression test for golang.org/issue/34499: version command should not crash
// when executed in a deleted directory on Linux.
func TestExecInDeletedDir(t *testing.T) {
switch runtime.GOOS {
case "windows", "plan9",
"aix", // Fails with "device busy".
"solaris", "illumos": // Fails with "invalid argument".
t.Skipf("%v does not support removing the current working directory", runtime.GOOS)
}
tg := testgo(t)
defer tg.cleanup()
wd, err := os.Getwd()
tg.check(err)
tg.makeTempdir()
tg.check(os.Chdir(tg.tempdir))
defer func() { tg.check(os.Chdir(wd)) }()
tg.check(os.Remove(tg.tempdir))
// `go version` should not fail
tg.run("version")
}
// A missing C compiler should not force the net package to be stale.
// Issue 47215.
func TestMissingCC(t *testing.T) {
if !canCgo {
t.Skip("test is only meaningful on systems with cgo")
}
cc := os.Getenv("CC")
if cc == "" {
cc = "gcc"
}
if filepath.IsAbs(cc) {
t.Skipf(`"CC" (%s) is an absolute path`, cc)
}
_, err := exec.LookPath(cc)
if err != nil {
t.Skipf(`"CC" (%s) not on PATH`, cc)
}
tg := testgo(t)
defer tg.cleanup()
netStale, _ := tg.isStale("net")
if netStale {
t.Skip(`skipping test because "net" package is currently stale`)
}
tg.setenv("PATH", "") // No C compiler on PATH.
netStale, _ = tg.isStale("net")
if netStale {
t.Error(`clearing "PATH" causes "net" to be stale`)
}
}

1
libgo/go/cmd/go/go_unix_test.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build darwin || dragonfly || freebsd || hurd || linux || netbsd || openbsd || solaris
// +build darwin dragonfly freebsd hurd linux netbsd openbsd solaris
package main_test

28
libgo/go/cmd/go/internal/base/path.go
View file

@ -8,21 +8,27 @@ import (
"os"
"path/filepath"
"strings"
"sync"
)
func getwd() string {
wd, err := os.Getwd()
if err != nil {
Fatalf("cannot determine current directory: %v", err)
}
return wd
}
var cwd string
var cwdOnce sync.Once
var Cwd = getwd()
// Cwd returns the current working directory at the time of the first call.
func Cwd() string {
cwdOnce.Do(func() {
var err error
cwd, err = os.Getwd()
if err != nil {
Fatalf("cannot determine current directory: %v", err)
}
})
return cwd
}
// ShortPath returns an absolute or relative name for path, whatever is shorter.
func ShortPath(path string) string {
if rel, err := filepath.Rel(Cwd, path); err == nil && len(rel) < len(path) {
if rel, err := filepath.Rel(Cwd(), path); err == nil && len(rel) < len(path) {
return rel
}
return path
@ -32,10 +38,8 @@ func ShortPath(path string) string {
// made relative to the current directory if they would be shorter.
func RelPaths(paths []string) []string {
var out []string
// TODO(rsc): Can this use Cwd from above?
pwd, _ := os.Getwd()
for _, p := range paths {
rel, err := filepath.Rel(pwd, p)
rel, err := filepath.Rel(Cwd(), p)
if err == nil && len(rel) < len(p) {
p = rel
}

1
libgo/go/cmd/go/internal/base/signal_notunix.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build plan9 || windows
// +build plan9 windows
package base

1
libgo/go/cmd/go/internal/base/signal_unix.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build aix || darwin || dragonfly || freebsd || hurd || js || linux || netbsd || openbsd || solaris
// +build aix darwin dragonfly freebsd hurd js linux netbsd openbsd solaris
package base

12
libgo/go/cmd/go/internal/bug/bug.go
View file

@ -20,6 +20,7 @@ import (
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/envcmd"
"cmd/go/internal/web"
)
@ -81,7 +82,7 @@ func printGoVersion(w io.Writer) {
fmt.Fprintf(w, "### What version of Go are you using (`go version`)?\n\n")
fmt.Fprintf(w, "<pre>\n")
fmt.Fprintf(w, "$ go version\n")
printCmdOut(w, "", "go", "version")
fmt.Fprintf(w, "go version %s %s/%s\n", runtime.Version(), runtime.GOOS, runtime.GOARCH)
fmt.Fprintf(w, "</pre>\n")
fmt.Fprintf(w, "\n")
}
@ -90,13 +91,20 @@ func printEnvDetails(w io.Writer) {
fmt.Fprintf(w, "### What operating system and processor architecture are you using (`go env`)?\n\n")
fmt.Fprintf(w, "<details><summary><code>go env</code> Output</summary><br><pre>\n")
fmt.Fprintf(w, "$ go env\n")
printCmdOut(w, "", "go", "env")
printGoEnv(w)
printGoDetails(w)
printOSDetails(w)
printCDetails(w)
fmt.Fprintf(w, "</pre></details>\n\n")
}
func printGoEnv(w io.Writer) {
env := envcmd.MkEnv()
env = append(env, envcmd.ExtraEnvVars()...)
env = append(env, envcmd.ExtraEnvVarsCostly()...)
envcmd.PrintEnv(w, env)
}
func printGoDetails(w io.Writer) {
printCmdOut(w, "GOROOT/bin/go version: ", filepath.Join(runtime.GOROOT(), "bin/go"), "version")
printCmdOut(w, "GOROOT/bin/go tool compile -V: ", filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "compile", "-V")

23
libgo/go/cmd/go/internal/cache/cache.go vendored
View file

@ -19,7 +19,7 @@ import (
"strings"
"time"
"cmd/go/internal/renameio"
"cmd/go/internal/lockedfile"
)
// An ActionID is a cache action key, the hash of a complete description of a
@ -294,10 +294,17 @@ func (c *Cache) Trim() {
// We maintain in dir/trim.txt the time of the last completed cache trim.
// If the cache has been trimmed recently enough, do nothing.
// This is the common case.
data, _ := renameio.ReadFile(filepath.Join(c.dir, "trim.txt"))
t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
if err == nil && now.Sub(time.Unix(t, 0)) < trimInterval {
return
// If the trim file is corrupt, detected if the file can't be parsed, or the
// trim time is too far in the future, attempt the trim anyway. It's possible that
// the cache was full when the corruption happened. Attempting a trim on
// an empty cache is cheap, so there wouldn't be a big performance hit in that case.
if data, err := lockedfile.Read(filepath.Join(c.dir, "trim.txt")); err == nil {
if t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); err == nil {
lastTrim := time.Unix(t, 0)
if d := now.Sub(lastTrim); d < trimInterval && d > -mtimeInterval {
return
}
}
}
// Trim each of the 256 subdirectories.
@ -311,7 +318,11 @@ func (c *Cache) Trim() {
// Ignore errors from here: if we don't write the complete timestamp, the
// cache will appear older than it is, and we'll trim it again next time.
renameio.WriteFile(filepath.Join(c.dir, "trim.txt"), []byte(fmt.Sprintf("%d", now.Unix())), 0666)
var b bytes.Buffer
fmt.Fprintf(&b, "%d", now.Unix())
if err := lockedfile.Write(filepath.Join(c.dir, "trim.txt"), &b, 0666); err != nil {
return
}
}
// trimSubdir trims a single cache subdirectory.

18
libgo/go/cmd/go/internal/cache/hash.go vendored
View file

@ -12,6 +12,7 @@ import (
"io"
"os"
"runtime"
"strings"
"sync"
)
@ -36,7 +37,22 @@ type Hash struct {
// of other versions. This salt will result in additional ActionID files
// in the cache, but not additional copies of the large output files,
// which are still addressed by unsalted SHA256.
var hashSalt = []byte(runtime.Version())
//
// We strip any GOEXPERIMENTs the go tool was built with from this
// version string on the assumption that they shouldn't affect go tool
// execution. This allows bootstrapping to converge faster: dist builds
// go_bootstrap without any experiments, so by stripping experiments
// go_bootstrap and the final go binary will use the same salt.
var hashSalt = []byte(stripExperiment(runtime.Version()))
// stripExperiment strips any GOEXPERIMENT configuration from the Go
// version string.
func stripExperiment(version string) string {
if i := strings.Index(version, " X:"); i >= 0 {
return version[:i]
}
return version
}
// Subkey returns an action ID corresponding to mixing a parent
// action ID with a string description of the subkey.

49
libgo/go/cmd/go/internal/cfg/cfg.go
View file

@ -10,6 +10,7 @@ import (
"bytes"
"fmt"
"go/build"
"internal/buildcfg"
"internal/cfg"
"io"
"os"
@ -19,8 +20,6 @@ import (
"sync"
"cmd/go/internal/fsys"
"cmd/internal/objabi"
)
// These are general "build flags" used by build and other commands.
@ -28,18 +27,18 @@ var (
BuildA bool // -a flag
BuildBuildmode string // -buildmode flag
BuildContext = defaultContext()
BuildMod string // -mod flag
BuildModExplicit bool // whether -mod was set explicitly
BuildModReason string // reason -mod was set, if set by default
BuildI bool // -i flag
BuildLinkshared bool // -linkshared flag
BuildMSan bool // -msan flag
BuildN bool // -n flag
BuildO string // -o flag
BuildP = runtime.NumCPU() // -p flag
BuildPkgdir string // -pkgdir flag
BuildRace bool // -race flag
BuildToolexec []string // -toolexec flag
BuildMod string // -mod flag
BuildModExplicit bool // whether -mod was set explicitly
BuildModReason string // reason -mod was set, if set by default
BuildI bool // -i flag
BuildLinkshared bool // -linkshared flag
BuildMSan bool // -msan flag
BuildN bool // -n flag
BuildO string // -o flag
BuildP = runtime.GOMAXPROCS(0) // -p flag
BuildPkgdir string // -pkgdir flag
BuildRace bool // -race flag
BuildToolexec []string // -toolexec flag
BuildToolchainName string
BuildToolchainCompiler func() string
BuildToolchainLinker func() string
@ -51,8 +50,6 @@ var (
ModCacheRW bool // -modcacherw flag
ModFile string // -modfile flag
Insecure bool // -insecure flag
CmdName string // "build", "install", "list", "mod tidy", etc.
DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable)
@ -80,6 +77,14 @@ func defaultContext() build.Context {
ctxt.GOOS = envOr("GOOS", ctxt.GOOS)
ctxt.GOARCH = envOr("GOARCH", ctxt.GOARCH)
// The experiments flags are based on GOARCH, so they may
// need to change. TODO: This should be cleaned up.
buildcfg.UpdateExperiments(ctxt.GOOS, ctxt.GOARCH, envOr("GOEXPERIMENT", buildcfg.DefaultGOEXPERIMENT))
ctxt.ToolTags = nil
for _, exp := range buildcfg.EnabledExperiments() {
ctxt.ToolTags = append(ctxt.ToolTags, "goexperiment."+exp)
}
// The go/build rule for whether cgo is enabled is:
// 1. If $CGO_ENABLED is set, respect it.
// 2. Otherwise, if this is a cross-compile, disable cgo.
@ -254,12 +259,12 @@ var (
GOMODCACHE = envOr("GOMODCACHE", gopathDir("pkg/mod"))
// Used in envcmd.MkEnv and build ID computations.
GOARM = envOr("GOARM", fmt.Sprint(objabi.GOARM))
GO386 = envOr("GO386", objabi.GO386)
GOMIPS = envOr("GOMIPS", objabi.GOMIPS)
GOMIPS64 = envOr("GOMIPS64", objabi.GOMIPS64)
GOPPC64 = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", objabi.GOPPC64))
GOWASM = envOr("GOWASM", fmt.Sprint(objabi.GOWASM))
GOARM = envOr("GOARM", fmt.Sprint(buildcfg.GOARM))
GO386 = envOr("GO386", buildcfg.GO386)
GOMIPS = envOr("GOMIPS", buildcfg.GOMIPS)
GOMIPS64 = envOr("GOMIPS64", buildcfg.GOMIPS64)
GOPPC64 = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", buildcfg.GOPPC64))
GOWASM = envOr("GOWASM", fmt.Sprint(buildcfg.GOWASM))
GOPROXY = envOr("GOPROXY", "https://proxy.golang.org,direct")
GOSUMDB = envOr("GOSUMDB", "sum.golang.org")

2
libgo/go/cmd/go/internal/clean/clean.go
View file

@ -117,7 +117,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) {
}
if cleanPkg {
for _, pkg := range load.PackagesAndErrors(ctx, args) {
for _, pkg := range load.PackagesAndErrors(ctx, load.PackageOpts{}, args) {
clean(pkg)
}
}

2
libgo/go/cmd/go/internal/doc/doc.go
View file

@ -13,7 +13,7 @@ import (
var CmdDoc = &base.Command{
Run: runDoc,
UsageLine: "go doc [-u] [-c] [package|[package.]symbol[.methodOrField]]",
UsageLine: "go doc [doc flags] [package|[package.]symbol[.methodOrField]]",
CustomFlags: true,
Short: "show documentation for package or symbol",
Long: `

242
libgo/go/cmd/go/internal/envcmd/env.go
View file

@ -10,6 +10,8 @@ import (
"encoding/json"
"fmt"
"go/build"
"internal/buildcfg"
"io"
"os"
"path/filepath"
"runtime"
@ -71,6 +73,7 @@ func MkEnv() []cfg.EnvVar {
{Name: "GOCACHE", Value: cache.DefaultDir()},
{Name: "GOENV", Value: envFile},
{Name: "GOEXE", Value: cfg.ExeSuffix},
{Name: "GOEXPERIMENT", Value: buildcfg.GOEXPERIMENT()},
{Name: "GOFLAGS", Value: cfg.Getenv("GOFLAGS")},
{Name: "GOHOSTARCH", Value: runtime.GOARCH},
{Name: "GOHOSTOS", Value: runtime.GOOS},
@ -196,23 +199,31 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) {
if *envU && *envW {
base.Fatalf("go env: cannot use -u with -w")
}
// Handle 'go env -w' and 'go env -u' before calling buildcfg.Check,
// so they can be used to recover from an invalid configuration.
if *envW {
runEnvW(args)
return
}
if *envU {
runEnvU(args)
return
}
buildcfg.Check()
env := cfg.CmdEnv
env = append(env, ExtraEnvVars()...)
if err := fsys.Init(base.Cwd); err != nil {
if err := fsys.Init(base.Cwd()); err != nil {
base.Fatalf("go: %v", err)
}
// Do we need to call ExtraEnvVarsCostly, which is a bit expensive?
needCostly := false
if *envU || *envW {
// We're overwriting or removing default settings,
// so it doesn't really matter what the existing settings are.
//
// Moreover, we haven't validated the new settings yet, so it is
// important that we NOT perform any actions based on them,
// such as initializing the builder to compute other variables.
} else if len(args) == 0 {
if len(args) == 0 {
// We're listing all environment variables ("go env"),
// including the expensive ones.
needCostly = true
@ -237,95 +248,6 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) {
env = append(env, ExtraEnvVarsCostly()...)
}
if *envW {
// Process and sanity-check command line.
if len(args) == 0 {
base.Fatalf("go env -w: no KEY=VALUE arguments given")
}
osEnv := make(map[string]string)
for _, e := range cfg.OrigEnv {
if i := strings.Index(e, "="); i >= 0 {
osEnv[e[:i]] = e[i+1:]
}
}
add := make(map[string]string)
for _, arg := range args {
i := strings.Index(arg, "=")
if i < 0 {
base.Fatalf("go env -w: arguments must be KEY=VALUE: invalid argument: %s", arg)
}
key, val := arg[:i], arg[i+1:]
if err := checkEnvWrite(key, val); err != nil {
base.Fatalf("go env -w: %v", err)
}
if _, ok := add[key]; ok {
base.Fatalf("go env -w: multiple values for key: %s", key)
}
add[key] = val
if osVal := osEnv[key]; osVal != "" && osVal != val {
fmt.Fprintf(os.Stderr, "warning: go env -w %s=... does not override conflicting OS environment variable\n", key)
}
}
goos, okGOOS := add["GOOS"]
goarch, okGOARCH := add["GOARCH"]
if okGOOS || okGOARCH {
if !okGOOS {
goos = cfg.Goos
}
if !okGOARCH {
goarch = cfg.Goarch
}
if err := work.CheckGOOSARCHPair(goos, goarch); err != nil {
base.Fatalf("go env -w: %v", err)
}
}
gotmp, okGOTMP := add["GOTMPDIR"]
if okGOTMP {
if !filepath.IsAbs(gotmp) && gotmp != "" {
base.Fatalf("go env -w: GOTMPDIR must be an absolute path")
}
}
updateEnvFile(add, nil)
return
}
if *envU {
// Process and sanity-check command line.
if len(args) == 0 {
base.Fatalf("go env -u: no arguments given")
}
del := make(map[string]bool)
for _, arg := range args {
if err := checkEnvWrite(arg, ""); err != nil {
base.Fatalf("go env -u: %v", err)
}
del[arg] = true
}
if del["GOOS"] || del["GOARCH"] {
goos, goarch := cfg.Goos, cfg.Goarch
if del["GOOS"] {
goos = getOrigEnv("GOOS")
if goos == "" {
goos = build.Default.GOOS
}
}
if del["GOARCH"] {
goarch = getOrigEnv("GOARCH")
if goarch == "" {
goarch = build.Default.GOARCH
}
}
if err := work.CheckGOOSARCHPair(goos, goarch); err != nil {
base.Fatalf("go env -u: %v", err)
}
}
updateEnvFile(nil, del)
return
}
if len(args) > 0 {
if *envJson {
var es []cfg.EnvVar
@ -347,27 +269,135 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) {
return
}
PrintEnv(os.Stdout, env)
}
func runEnvW(args []string) {
// Process and sanity-check command line.
if len(args) == 0 {
base.Fatalf("go env -w: no KEY=VALUE arguments given")
}
osEnv := make(map[string]string)
for _, e := range cfg.OrigEnv {
if i := strings.Index(e, "="); i >= 0 {
osEnv[e[:i]] = e[i+1:]
}
}
add := make(map[string]string)
for _, arg := range args {
i := strings.Index(arg, "=")
if i < 0 {
base.Fatalf("go env -w: arguments must be KEY=VALUE: invalid argument: %s", arg)
}
key, val := arg[:i], arg[i+1:]
if err := checkEnvWrite(key, val); err != nil {
base.Fatalf("go env -w: %v", err)
}
if _, ok := add[key]; ok {
base.Fatalf("go env -w: multiple values for key: %s", key)
}
add[key] = val
if osVal := osEnv[key]; osVal != "" && osVal != val {
fmt.Fprintf(os.Stderr, "warning: go env -w %s=... does not override conflicting OS environment variable\n", key)
}
}
if err := checkBuildConfig(add, nil); err != nil {
base.Fatalf("go env -w: %v", err)
}
gotmp, okGOTMP := add["GOTMPDIR"]
if okGOTMP {
if !filepath.IsAbs(gotmp) && gotmp != "" {
base.Fatalf("go env -w: GOTMPDIR must be an absolute path")
}
}
updateEnvFile(add, nil)
}
func runEnvU(args []string) {
// Process and sanity-check command line.
if len(args) == 0 {
base.Fatalf("go env -u: no arguments given")
}
del := make(map[string]bool)
for _, arg := range args {
if err := checkEnvWrite(arg, ""); err != nil {
base.Fatalf("go env -u: %v", err)
}
del[arg] = true
}
if err := checkBuildConfig(nil, del); err != nil {
base.Fatalf("go env -u: %v", err)
}
updateEnvFile(nil, del)
}
// checkBuildConfig checks whether the build configuration is valid
// after the specified configuration environment changes are applied.
func checkBuildConfig(add map[string]string, del map[string]bool) error {
// get returns the value for key after applying add and del and
// reports whether it changed. cur should be the current value
// (i.e., before applying changes) and def should be the default
// value (i.e., when no environment variables are provided at all).
get := func(key, cur, def string) (string, bool) {
if val, ok := add[key]; ok {
return val, true
}
if del[key] {
val := getOrigEnv(key)
if val == "" {
val = def
}
return val, true
}
return cur, false
}
goos, okGOOS := get("GOOS", cfg.Goos, build.Default.GOOS)
goarch, okGOARCH := get("GOARCH", cfg.Goarch, build.Default.GOARCH)
if okGOOS || okGOARCH {
if err := work.CheckGOOSARCHPair(goos, goarch); err != nil {
return err
}
}
goexperiment, okGOEXPERIMENT := get("GOEXPERIMENT", buildcfg.GOEXPERIMENT(), "")
if okGOEXPERIMENT {
if _, _, err := buildcfg.ParseGOEXPERIMENT(goos, goarch, goexperiment); err != nil {
return err
}
}
return nil
}
// PrintEnv prints the environment variables to w.
func PrintEnv(w io.Writer, env []cfg.EnvVar) {
for _, e := range env {
if e.Name != "TERM" {
switch runtime.GOOS {
default:
fmt.Printf("%s=\"%s\"\n", e.Name, e.Value)
fmt.Fprintf(w, "%s=\"%s\"\n", e.Name, e.Value)
case "plan9":
if strings.IndexByte(e.Value, '\x00') < 0 {
fmt.Printf("%s='%s'\n", e.Name, strings.ReplaceAll(e.Value, "'", "''"))
fmt.Fprintf(w, "%s='%s'\n", e.Name, strings.ReplaceAll(e.Value, "'", "''"))
} else {
v := strings.Split(e.Value, "\x00")
fmt.Printf("%s=(", e.Name)
fmt.Fprintf(w, "%s=(", e.Name)
for x, s := range v {
if x > 0 {
fmt.Printf(" ")
fmt.Fprintf(w, " ")
}
fmt.Printf("%s", s)
fmt.Fprintf(w, "%s", s)
}
fmt.Printf(")\n")
fmt.Fprintf(w, ")\n")
}
case "windows":
fmt.Printf("set %s=%s\n", e.Name, e.Value)
fmt.Fprintf(w, "set %s=%s\n", e.Name, e.Value)
}
}
}
@ -428,7 +458,7 @@ func checkEnvWrite(key, val string) error {
return fmt.Errorf("GOPATH entry is relative; must be absolute path: %q", val)
}
// Make sure CC and CXX are absolute paths
case "CC", "CXX":
case "CC", "CXX", "GOMODCACHE":
if !filepath.IsAbs(val) && val != "" && val != filepath.Base(val) {
return fmt.Errorf("%s entry is relative; must be absolute path: %q", key, val)
}

2
libgo/go/cmd/go/internal/fix/fix.go
View file

@ -33,7 +33,7 @@ See also: go fmt, go vet.
}
func runFix(ctx context.Context, cmd *base.Command, args []string) {
pkgs := load.PackagesAndErrors(ctx, args)
pkgs := load.PackagesAndErrors(ctx, load.PackageOpts{}, args)
w := 0
for _, pkg := range pkgs {
if pkg.Error != nil {

2
libgo/go/cmd/go/internal/fmtcmd/fmt.go
View file

@ -65,7 +65,7 @@ func runFmt(ctx context.Context, cmd *base.Command, args []string) {
}
}()
}
for _, pkg := range load.PackagesAndErrors(ctx, args) {
for _, pkg := range load.PackagesAndErrors(ctx, load.PackageOpts{}, args) {
if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main {
if !printed {
fmt.Fprintf(os.Stderr, "go: not formatting packages in dependency modules\n")

4
libgo/go/cmd/go/internal/fsys/fsys.go
View file

@ -44,7 +44,7 @@ func (n *node) isDeleted() bool {
// TODO(matloob): encapsulate these in an io/fs-like interface
var overlay map[string]*node // path -> file or directory node
var cwd string // copy of base.Cwd to avoid dependency
var cwd string // copy of base.Cwd() to avoid dependency
// Canonicalize a path for looking it up in the overlay.
// Important: filepath.Join(cwd, path) doesn't always produce
@ -100,7 +100,7 @@ func Init(wd string) error {
}
func initFromJSON(overlayJSON OverlayJSON) error {
// Canonicalize the paths in in the overlay map.
// Canonicalize the paths in the overlay map.
// Use reverseCanonicalized to check for collisions:
// no two 'from' paths should canonicalize to the same path.
overlay = make(map[string]*node)

6
libgo/go/cmd/go/internal/generate/generate.go
View file

@ -161,8 +161,6 @@ func init() {
}
func runGenerate(ctx context.Context, cmd *base.Command, args []string) {
load.IgnoreImports = true
if generateRunFlag != "" {
var err error
generateRunRE, err = regexp.Compile(generateRunFlag)
@ -175,7 +173,8 @@ func runGenerate(ctx context.Context, cmd *base.Command, args []string) {
// Even if the arguments are .go files, this loop suffices.
printed := false
for _, pkg := range load.PackagesAndErrors(ctx, args) {
pkgOpts := load.PackageOpts{IgnoreImports: true}
for _, pkg := range load.PackagesAndErrors(ctx, pkgOpts, args) {
if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main {
if !printed {
fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n")
@ -334,6 +333,7 @@ func (g *Generator) setEnv() {
"GOPACKAGE=" + g.pkg,
"DOLLAR=" + "$",
}
g.env = base.AppendPWD(g.env, g.dir)
}
// split breaks the line into words, evaluating quoted

49
libgo/go/cmd/go/internal/get/get.go
View file

@ -26,7 +26,7 @@ import (
)
var CmdGet = &base.Command{
UsageLine: "go get [-d] [-f] [-t] [-u] [-v] [-fix] [-insecure] [build flags] [packages]",
UsageLine: "go get [-d] [-f] [-t] [-u] [-v] [-fix] [build flags] [packages]",
Short: "download and install packages and dependencies",
Long: `
Get downloads the packages named by the import paths, along with their
@ -43,13 +43,6 @@ of the original.
The -fix flag instructs get to run the fix tool on the downloaded packages
before resolving dependencies or building the code.
The -insecure flag permits fetching from repositories and resolving
custom domains using insecure schemes such as HTTP. Use with caution.
This flag is deprecated and will be removed in a future version of go.
The GOINSECURE environment variable should be used instead, since it
provides control over which packages may be retrieved using an insecure
scheme. See 'go help environment' for details.
The -t flag instructs get to also download the packages required to build
the tests for the specified packages.
@ -105,17 +98,17 @@ Usage: ` + CmdGet.UsageLine + `
}
var (
getD = CmdGet.Flag.Bool("d", false, "")
getF = CmdGet.Flag.Bool("f", false, "")
getT = CmdGet.Flag.Bool("t", false, "")
getU = CmdGet.Flag.Bool("u", false, "")
getFix = CmdGet.Flag.Bool("fix", false, "")
getD = CmdGet.Flag.Bool("d", false, "")
getF = CmdGet.Flag.Bool("f", false, "")
getT = CmdGet.Flag.Bool("t", false, "")
getU = CmdGet.Flag.Bool("u", false, "")
getFix = CmdGet.Flag.Bool("fix", false, "")
getInsecure = CmdGet.Flag.Bool("insecure", false, "")
)
func init() {
work.AddBuildFlags(CmdGet, work.OmitModFlag|work.OmitModCommonFlags)
CmdGet.Run = runGet // break init loop
CmdGet.Flag.BoolVar(&cfg.Insecure, "insecure", cfg.Insecure, "")
}
func runGet(ctx context.Context, cmd *base.Command, args []string) {
@ -129,11 +122,11 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
if *getF && !*getU {
base.Fatalf("go get: cannot use -f flag without -u")
}
if cfg.Insecure {
fmt.Fprintf(os.Stderr, "go get: -insecure flag is deprecated; see 'go help get' for details\n")
if *getInsecure {
base.Fatalf("go get: -insecure flag is no longer supported; use GOINSECURE instead")
}
// Disable any prompting for passwords by Git.
// Disable any prompting for passwords by Git itself.
// Only has an effect for 2.3.0 or later, but avoiding
// the prompt in earlier versions is just too hard.
// If user has explicitly set GIT_TERMINAL_PROMPT=1, keep
@ -143,7 +136,10 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
os.Setenv("GIT_TERMINAL_PROMPT", "0")
}
// Disable any ssh connection pooling by Git.
// Also disable prompting for passwords by the 'ssh' subprocess spawned by
// Git, because apparently GIT_TERMINAL_PROMPT isn't sufficient to do that.
// Adding '-o BatchMode=yes' should do the trick.
//
// If a Git subprocess forks a child into the background to cache a new connection,
// that child keeps stdout/stderr open. After the Git subprocess exits,
// os /exec expects to be able to read from the stdout/stderr pipe
@ -157,7 +153,14 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
// assume they know what they are doing and don't step on it.
// But default to turning off ControlMaster.
if os.Getenv("GIT_SSH") == "" && os.Getenv("GIT_SSH_COMMAND") == "" {
os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no")
os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no -o BatchMode=yes")
}
// And one more source of Git prompts: the Git Credential Manager Core for Windows.
//
// See https://github.com/microsoft/Git-Credential-Manager-Core/blob/master/docs/environment.md#gcm_interactive.
if os.Getenv("GCM_INTERACTIVE") == "" {
os.Setenv("GCM_INTERACTIVE", "never")
}
// Phase 1. Download/update.
@ -180,7 +183,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
// everything.
load.ClearPackageCache()
pkgs := load.PackagesAndErrors(ctx, args)
pkgs := load.PackagesAndErrors(ctx, load.PackageOpts{}, args)
load.CheckPackageErrors(pkgs)
// Phase 3. Install.
@ -255,9 +258,9 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int)
load1 := func(path string, mode int) *load.Package {
if parent == nil {
mode := 0 // don't do module or vendor resolution
return load.LoadImport(context.TODO(), path, base.Cwd, nil, stk, nil, mode)
return load.LoadImport(context.TODO(), load.PackageOpts{}, path, base.Cwd(), nil, stk, nil, mode)
}
return load.LoadImport(context.TODO(), path, parent.Dir, parent, stk, nil, mode|load.ResolveModule)
return load.LoadImport(context.TODO(), load.PackageOpts{}, path, parent.Dir, parent, stk, nil, mode|load.ResolveModule)
}
p := load1(arg, mode)
@ -435,7 +438,7 @@ func downloadPackage(p *load.Package) error {
return fmt.Errorf("%s: invalid import path: %v", p.ImportPath, err)
}
security := web.SecureOnly
if cfg.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, importPrefix) {
if module.MatchPrefixPatterns(cfg.GOINSECURE, importPrefix) {
security = web.Insecure
}

58
libgo/go/cmd/go/internal/help/helpdoc.go
View file

@ -251,7 +251,7 @@ For example,
will result in the following requests:
https://example.org/pkg/foo?go-get=1 (preferred)
http://example.org/pkg/foo?go-get=1 (fallback, only with -insecure)
http://example.org/pkg/foo?go-get=1 (fallback, only with use of correctly set GOINSECURE)
If that page contains the meta tag
@ -517,9 +517,8 @@ General-purpose environment variables:
Comma-separated list of glob patterns (in the syntax of Go's path.Match)
of module path prefixes that should always be fetched in an insecure
manner. Only applies to dependencies that are being fetched directly.
Unlike the -insecure flag on 'go get', GOINSECURE does not disable
checksum database validation. GOPRIVATE or GONOSUMDB may be used
to achieve that.
GOINSECURE does not disable checksum database validation. GOPRIVATE or
GONOSUMDB may be used to achieve that.
GOOS
The operating system for which to compile code.
Examples are linux, darwin, windows, netbsd.
@ -599,6 +598,9 @@ Architecture-specific environment variables:
GOMIPS64
For GOARCH=mips64{,le}, whether to use floating point instructions.
Valid values are hardfloat (default), softfloat.
GOPPC64
For GOARCH=ppc64{,le}, the target ISA (Instruction Set Architecture).
Valid values are power8 (default), power9.
GOWASM
For GOARCH=wasm, comma-separated list of experimental WebAssembly features to use.
Valid values are satconv, signext.
@ -608,6 +610,12 @@ Special-purpose environment variables:
GCCGOTOOLDIR
If set, where to find gccgo tools, such as cgo.
The default is based on how gccgo was configured.
GOEXPERIMENT
Comma-separated list of toolchain experiments to enable or disable.
The list of available experiments may change arbitrarily over time.
See src/internal/goexperiment/flags.go for currently valid values.
Warning: This variable is provided for the development and testing
of the Go toolchain itself. Use beyond that purpose is unsupported.
GOROOT_FINAL
The root of the installed Go tree, when it is
installed in a location other than where it is built.
@ -785,7 +793,7 @@ var HelpBuildConstraint = &base.Command{
Long: `
A build constraint, also known as a build tag, is a line comment that begins
// +build
//go:build
that lists the conditions under which a file should be included in the package.
Constraints may appear in any kind of source file (not just Go), but
@ -793,30 +801,20 @@ they must appear near the top of the file, preceded
only by blank lines and other line comments. These rules mean that in Go
files a build constraint must appear before the package clause.
To distinguish build constraints from package documentation, a series of
build constraints must be followed by a blank line.
To distinguish build constraints from package documentation,
a build constraint should be followed by a blank line.
A build constraint is evaluated as the OR of space-separated options.
Each option evaluates as the AND of its comma-separated terms.
Each term consists of letters, digits, underscores, and dots.
A term may be negated with a preceding !.
For example, the build constraint:
A build constraint is evaluated as an expression containing options
combined by ||, &&, and ! operators and parentheses. Operators have
the same meaning as in Go.
// +build linux,386 darwin,!cgo
For example, the following build constraint constrains a file to
build when the "linux" and "386" constraints are satisfied, or when
"darwin" is satisfied and "cgo" is not:
corresponds to the boolean formula:
//go:build (linux && 386) || (darwin && !cgo)
(linux AND 386) OR (darwin AND (NOT cgo))
A file may have multiple build constraints. The overall constraint is the AND
of the individual constraints. That is, the build constraints:
// +build linux darwin
// +build amd64
corresponds to the boolean formula:
(linux OR darwin) AND amd64
It is an error for a file to have more than one //go:build line.
During a particular build, the following words are satisfied:
@ -854,22 +852,26 @@ in addition to ios tags and files.
To keep a file from being considered for the build:
// +build ignore
//go:build ignore
(any other unsatisfied word will work as well, but "ignore" is conventional.)
To build a file only when using cgo, and only on Linux and OS X:
// +build linux,cgo darwin,cgo
//go:build cgo && (linux || darwin)
Such a file is usually paired with another file implementing the
default functionality for other systems, which in this case would
carry the constraint:
// +build !linux,!darwin !cgo
//go:build !(cgo && (linux || darwin))
Naming a file dns_windows.go will cause it to be included only when
building the package for Windows; similarly, math_386.s will be included
only when building the package for 32-bit x86.
Go versions 1.16 and earlier used a different syntax for build constraints,
with a "// +build" prefix. The gofmt command will add an equivalent //go:build
constraint when encountering the older syntax.
`,
}

18
libgo/go/cmd/go/internal/imports/read.go
View file

@ -8,6 +8,7 @@ package imports
import (
"bufio"
"bytes"
"errors"
"io"
"unicode/utf8"
@ -22,6 +23,19 @@ type importReader struct {
nerr int
}
var bom = []byte{0xef, 0xbb, 0xbf}
func newImportReader(b *bufio.Reader) *importReader {
// Remove leading UTF-8 BOM.
// Per https://golang.org/ref/spec#Source_code_representation:
// a compiler may ignore a UTF-8-encoded byte order mark (U+FEFF)
// if it is the first Unicode code point in the source text.
if leadingBytes, err := b.Peek(3); err == nil && bytes.Equal(leadingBytes, bom) {
b.Discard(3)
}
return &importReader{b: b}
}
func isIdent(c byte) bool {
return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '_' || c >= utf8.RuneSelf
}
@ -201,7 +215,7 @@ func (r *importReader) readImport(imports *[]string) {
// ReadComments is like io.ReadAll, except that it only reads the leading
// block of comments in the file.
func ReadComments(f io.Reader) ([]byte, error) {
r := &importReader{b: bufio.NewReader(f)}
r := newImportReader(bufio.NewReader(f))
r.peekByte(true)
if r.err == nil && !r.eof {
// Didn't reach EOF, so must have found a non-space byte. Remove it.
@ -213,7 +227,7 @@ func ReadComments(f io.Reader) ([]byte, error) {
// ReadImports is like io.ReadAll, except that it expects a Go file as input
// and stops reading the input once the imports have completed.
func ReadImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte, error) {
r := &importReader{b: bufio.NewReader(f)}
r := newImportReader(bufio.NewReader(f))
r.readKeyword("package")
r.readIdent()

26
libgo/go/cmd/go/internal/imports/read_test.go
View file

@ -66,6 +66,10 @@ var readImportsTests = []readTest{
`,
"",
},
{
"\ufeff𝔻" + `package p; import "x";ℙvar x = 1`,
"",
},
}
var readCommentsTests = []readTest{
@ -81,6 +85,10 @@ var readCommentsTests = []readTest{
`ℙpackage p; import . "x"`,
"",
},
{
"\ufeff𝔻" + `ℙpackage p; import . "x"`,
"",
},
{
`// foo
@ -90,6 +98,19 @@ var readCommentsTests = []readTest{
/*/ zot */
// asdf
ℙHello, world`,
"",
},
{
"\ufeff𝔻" + `// foo
/* bar */
/* quux */ // baz
/*/ zot */
// asdf
ℙHello, world`,
"",
@ -107,6 +128,11 @@ func testRead(t *testing.T, tests []readTest, read func(io.Reader) ([]byte, erro
in = tt.in[:j] + tt.in[j+len("ℙ"):]
testOut = tt.in[:j]
}
d := strings.Index(tt.in, "𝔻")
if d >= 0 {
in = in[:d] + in[d+len("𝔻"):]
testOut = testOut[d+len("𝔻"):]
}
r := strings.NewReader(in)
buf, err := read(r)
if err != nil {

1
libgo/go/cmd/go/internal/imports/testdata/android/tags.txt vendored Normal file
View file

@ -0,0 +1 @@
android arm64

6
libgo/go/cmd/go/internal/imports/testdata/android/want.txt vendored Normal file
View file

@ -0,0 +1,6 @@
a
b
c
d
e
f

1
libgo/go/cmd/go/internal/imports/testdata/illumos/tags.txt vendored Normal file
View file

@ -0,0 +1 @@
illumos amd64

6
libgo/go/cmd/go/internal/imports/testdata/illumos/want.txt vendored Normal file
View file

@ -0,0 +1,6 @@
a
b
c
d
e
f

1
libgo/go/cmd/go/internal/imports/testdata/star/tags.txt vendored Normal file
View file

@ -0,0 +1 @@
*

4
libgo/go/cmd/go/internal/imports/testdata/star/want.txt vendored Normal file
View file

@ -0,0 +1,4 @@
import1
import2
import3
import4

2
libgo/go/cmd/go/internal/list/context.go
View file

@ -17,6 +17,7 @@ type Context struct {
UseAllFiles bool `json:",omitempty"` // use files regardless of +build lines, file names
Compiler string `json:",omitempty"` // compiler to assume when computing target paths
BuildTags []string `json:",omitempty"` // build constraints to match in +build lines
ToolTags []string `json:",omitempty"` // toolchain-specific build constraints
ReleaseTags []string `json:",omitempty"` // releases the current release is compatible with
InstallSuffix string `json:",omitempty"` // suffix to use in the name of the install dir
}
@ -31,6 +32,7 @@ func newContext(c *build.Context) *Context {
UseAllFiles: c.UseAllFiles,
Compiler: c.Compiler,
BuildTags: c.BuildTags,
ToolTags: c.ToolTags,
ReleaseTags: c.ReleaseTags,
InstallSuffix: c.InstallSuffix,
}

66
libgo/go/cmd/go/internal/list/list.go
View file

@ -148,6 +148,7 @@ The template function "context" returns the build context, defined as:
UseAllFiles bool // use files regardless of +build lines, file names
Compiler string // compiler to assume when computing target paths
BuildTags []string // build constraints to match in +build lines
ToolTags []string // toolchain-specific build constraints
ReleaseTags []string // releases the current release is compatible with
InstallSuffix string // suffix to use in the name of the install dir
}
@ -335,7 +336,10 @@ var (
var nl = []byte{'\n'}
func runList(ctx context.Context, cmd *base.Command, args []string) {
load.ModResolveTests = *listTest
if *listFmt != "" && *listJson == true {
base.Fatalf("go list -f cannot be used with -json")
}
work.BuildInit()
out := newTrackingWriter(os.Stdout)
defer out.w.Flush()
@ -344,7 +348,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
if *listM {
*listFmt = "{{.String}}"
if *listVersions {
*listFmt = `{{.Path}}{{range .Versions}} {{.}}{{end}}`
*listFmt = `{{.Path}}{{range .Versions}} {{.}}{{end}}{{if .Deprecated}} (deprecated){{end}}`
}
} else {
*listFmt = "{{.ImportPath}}"
@ -423,7 +427,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
base.Fatalf("go list -m: not using modules")
}
modload.LoadModFile(ctx) // Parses go.mod and sets cfg.BuildMod.
modload.LoadModFile(ctx) // Sets cfg.BuildMod as a side-effect.
if cfg.BuildMod == "vendor" {
const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)"
@ -447,13 +451,29 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
}
}
mods := modload.ListModules(ctx, args, *listU, *listVersions, *listRetracted)
var mode modload.ListMode
if *listU {
mode |= modload.ListU | modload.ListRetracted | modload.ListDeprecated
}
if *listRetracted {
mode |= modload.ListRetracted
}
if *listVersions {
mode |= modload.ListVersions
if *listRetracted {
mode |= modload.ListRetractedVersions
}
}
mods, err := modload.ListModules(ctx, args, mode)
if !*listE {
for _, m := range mods {
if m.Error != nil {
base.Errorf("go list -m: %v", m.Error.Err)
}
}
if err != nil {
base.Errorf("go list -m: %v", err)
}
base.ExitIfErrors()
}
for _, m := range mods {
@ -478,8 +498,11 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
base.Fatalf("go list -test cannot be used with -find")
}
load.IgnoreImports = *listFind
pkgs := load.PackagesAndErrors(ctx, args)
pkgOpts := load.PackageOpts{
IgnoreImports: *listFind,
ModResolveTests: *listTest,
}
pkgs := load.PackagesAndErrors(ctx, pkgOpts, args)
if !*listE {
w := 0
for _, pkg := range pkgs {
@ -516,9 +539,9 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
var pmain, ptest, pxtest *load.Package
var err error
if *listE {
pmain, ptest, pxtest = load.TestPackagesAndErrors(ctx, p, nil)
pmain, ptest, pxtest = load.TestPackagesAndErrors(ctx, pkgOpts, p, nil)
} else {
pmain, ptest, pxtest, err = load.TestPackagesFor(ctx, p, nil)
pmain, ptest, pxtest, err = load.TestPackagesFor(ctx, pkgOpts, p, nil)
if err != nil {
base.Errorf("can't load test package: %s", err)
}
@ -605,7 +628,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
old := make(map[string]string)
for _, p := range all {
if p.ForTest != "" {
new := p.ImportPath + " [" + p.ForTest + ".test]"
new := p.Desc()
old[new] = p.ImportPath
p.ImportPath = new
}
@ -679,9 +702,14 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
}
if len(args) > 0 {
listU := false
listVersions := false
rmods := modload.ListModules(ctx, args, listU, listVersions, *listRetracted)
var mode modload.ListMode
if *listRetracted {
mode |= modload.ListRetracted
}
rmods, err := modload.ListModules(ctx, args, mode)
if err != nil && !*listE {
base.Errorf("go list -retracted: %v", err)
}
for i, arg := range args {
rmod := rmods[i]
for _, mod := range argToMods[arg] {
@ -696,8 +724,18 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
// Record non-identity import mappings in p.ImportMap.
for _, p := range pkgs {
for i, srcPath := range p.Internal.RawImports {
path := p.Imports[i]
nRaw := len(p.Internal.RawImports)
for i, path := range p.Imports {
var srcPath string
if i < nRaw {
srcPath = p.Internal.RawImports[i]
} else {
// This path is not within the raw imports, so it must be an import
// found only within CompiledGoFiles. Those paths are found in
// CompiledImports.
srcPath = p.Internal.CompiledImports[i-nRaw]
}
if path != srcPath {
if p.ImportMap == nil {
p.ImportMap = make(map[string]string)

2
libgo/go/cmd/go/internal/load/flag.go
View file

@ -34,7 +34,7 @@ type ppfValue struct {
// Set is called each time the flag is encountered on the command line.
func (f *PerPackageFlag) Set(v string) error {
return f.set(v, base.Cwd)
return f.set(v, base.Cwd())
}
// set is the implementation of Set, taking a cwd (current working directory) for easier testing.

471
libgo/go/cmd/go/internal/load/pkg.go
View file

@ -14,6 +14,7 @@ import (
"go/build"
"go/scanner"
"go/token"
"internal/goroot"
"io/fs"
"os"
"path"
@ -29,6 +30,8 @@ import (
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/fsys"
"cmd/go/internal/imports"
"cmd/go/internal/modfetch"
"cmd/go/internal/modinfo"
"cmd/go/internal/modload"
"cmd/go/internal/par"
@ -37,11 +40,10 @@ import (
"cmd/go/internal/trace"
"cmd/internal/sys"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
var IgnoreImports bool // control whether we ignore imports in packages
// A Package describes a single package found in a directory.
type Package struct {
PackagePublic // visible in 'go list'
@ -85,6 +87,7 @@ type PackagePublic struct {
CgoFiles []string `json:",omitempty"` // .go source files that import "C"
CompiledGoFiles []string `json:",omitempty"` // .go output from running cgo on CgoFiles
IgnoredGoFiles []string `json:",omitempty"` // .go source files ignored due to build constraints
InvalidGoFiles []string `json:",omitempty"` // .go source files with detected problems (parse error, wrong package name, and so on)
IgnoredOtherFiles []string `json:",omitempty"` // non-.go source files ignored due to build constraints
CFiles []string `json:",omitempty"` // .c source files
CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files
@ -142,6 +145,7 @@ func (p *Package) AllFiles() []string {
p.CgoFiles,
// no p.CompiledGoFiles, because they are from GoFiles or generated by us
p.IgnoredGoFiles,
// no p.InvalidGoFiles, because they are from GoFiles
p.IgnoredOtherFiles,
p.CFiles,
p.CXXFiles,
@ -190,8 +194,8 @@ type PackageInternal struct {
// Unexported fields are not part of the public API.
Build *build.Package
Imports []*Package // this package's direct imports
CompiledImports []string // additional Imports necessary when using CompiledGoFiles (all from standard library)
RawImports []string // this package's original imports as they appear in the text of the program
CompiledImports []string // additional Imports necessary when using CompiledGoFiles (all from standard library); 1:1 with the end of PackagePublic.Imports
RawImports []string // this package's original imports as they appear in the text of the program; 1:1 with the end of PackagePublic.Imports
ForceLibrary bool // this package is a library (even if named "main")
CmdlineFiles bool // package built from files listed on command line
CmdlinePkg bool // package listed on command line
@ -206,6 +210,7 @@ type PackageInternal struct {
BuildInfo string // add this info to package main
TestmainGo *[]byte // content for _testmain.go
Embed map[string][]string // //go:embed comment mapping
OrigImportPath string // original import path before adding '_test' suffix
Asmflags []string // -asmflags for this package
Gcflags []string // -gcflags for this package
@ -343,7 +348,7 @@ type CoverVar struct {
Var string // name of count struct
}
func (p *Package) copyBuild(pp *build.Package) {
func (p *Package) copyBuild(opts PackageOpts, pp *build.Package) {
p.Internal.Build = pp
if pp.PkgTargetRoot != "" && cfg.BuildPkgdir != "" {
@ -368,6 +373,7 @@ func (p *Package) copyBuild(pp *build.Package) {
p.GoFiles = pp.GoFiles
p.CgoFiles = pp.CgoFiles
p.IgnoredGoFiles = pp.IgnoredGoFiles
p.InvalidGoFiles = pp.InvalidGoFiles
p.IgnoredOtherFiles = pp.IgnoredOtherFiles
p.CFiles = pp.CFiles
p.CXXFiles = pp.CXXFiles
@ -392,7 +398,7 @@ func (p *Package) copyBuild(pp *build.Package) {
p.TestImports = pp.TestImports
p.XTestGoFiles = pp.XTestGoFiles
p.XTestImports = pp.XTestImports
if IgnoreImports {
if opts.IgnoreImports {
p.Imports = nil
p.Internal.RawImports = nil
p.TestImports = nil
@ -401,6 +407,7 @@ func (p *Package) copyBuild(pp *build.Package) {
p.EmbedPatterns = pp.EmbedPatterns
p.TestEmbedPatterns = pp.TestEmbedPatterns
p.XTestEmbedPatterns = pp.XTestEmbedPatterns
p.Internal.OrigImportPath = pp.ImportPath
}
// A PackageError describes an error loading information about a package.
@ -476,8 +483,10 @@ type ImportPathError interface {
var (
_ ImportPathError = (*importError)(nil)
_ ImportPathError = (*mainPackageError)(nil)
_ ImportPathError = (*modload.ImportMissingError)(nil)
_ ImportPathError = (*modload.ImportMissingSumError)(nil)
_ ImportPathError = (*modload.DirectImportFromImplicitDependencyError)(nil)
)
type importError struct {
@ -597,7 +606,7 @@ func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package {
})
packageDataCache.Delete(p.ImportPath)
}
return LoadImport(context.TODO(), arg, base.Cwd, nil, stk, nil, 0)
return LoadImport(context.TODO(), PackageOpts{}, arg, base.Cwd(), nil, stk, nil, 0)
}
// dirToImportPath returns the pseudo-import path we use for a package
@ -649,11 +658,11 @@ const (
// LoadImport does not set tool flags and should only be used by
// this package, as part of a bigger load operation, and by GOPATH-based "go get".
// TODO(rsc): When GOPATH-based "go get" is removed, unexport this function.
func LoadImport(ctx context.Context, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
return loadImport(ctx, nil, path, srcDir, parent, stk, importPos, mode)
func LoadImport(ctx context.Context, opts PackageOpts, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
return loadImport(ctx, opts, nil, path, srcDir, parent, stk, importPos, mode)
}
func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
func loadImport(ctx context.Context, opts PackageOpts, pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
if path == "" {
panic("LoadImport called with empty package path")
}
@ -665,25 +674,30 @@ func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *
parentRoot = parent.Root
parentIsStd = parent.Standard
}
bp, loaded, err := loadPackageData(path, parentPath, srcDir, parentRoot, parentIsStd, mode)
if loaded && pre != nil && !IgnoreImports {
pre.preloadImports(bp.Imports, bp)
bp, loaded, err := loadPackageData(ctx, path, parentPath, srcDir, parentRoot, parentIsStd, mode)
if loaded && pre != nil && !opts.IgnoreImports {
pre.preloadImports(ctx, opts, bp.Imports, bp)
}
if bp == nil {
p := &Package{
PackagePublic: PackagePublic{
ImportPath: path,
Incomplete: true,
},
}
if importErr, ok := err.(ImportPathError); !ok || importErr.ImportPath() != path {
// Only add path to the error's import stack if it's not already present on the error.
// Only add path to the error's import stack if it's not already present
// in the error.
//
// TODO(bcmills): setLoadPackageDataError itself has a similar Push / Pop
// sequence that empirically doesn't trigger for these errors, guarded by
// a somewhat complex condition. Figure out how to generalize that
// condition and eliminate the explicit calls here.
stk.Push(path)
defer stk.Pop()
}
return &Package{
PackagePublic: PackagePublic{
ImportPath: path,
Error: &PackageError{
ImportStack: stk.Copy(),
Err: err,
},
},
}
p.setLoadPackageDataError(err, path, stk, nil)
return p
}
importPath := bp.ImportPath
@ -701,7 +715,7 @@ func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *
// Load package.
// loadPackageData may return bp != nil even if an error occurs,
// in order to return partial information.
p.load(ctx, path, stk, importPos, bp, err)
p.load(ctx, opts, path, stk, importPos, bp, err)
if !cfg.ModulesEnabled && path != cleanImport(path) {
p.Error = &PackageError{
@ -714,7 +728,7 @@ func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *
}
// Checked on every import because the rules depend on the code doing the importing.
if perr := disallowInternal(srcDir, parent, parentPath, p, stk); perr != p {
if perr := disallowInternal(ctx, srcDir, parent, parentPath, p, stk); perr != p {
perr.Error.setPos(importPos)
return perr
}
@ -763,7 +777,7 @@ func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *
//
// loadPackageData returns a boolean, loaded, which is true if this is the
// first time the package was loaded. Callers may preload imports in this case.
func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd bool, mode int) (bp *build.Package, loaded bool, err error) {
func loadPackageData(ctx context.Context, path, parentPath, parentDir, parentRoot string, parentIsStd bool, mode int) (bp *build.Package, loaded bool, err error) {
if path == "" {
panic("loadPackageData called with empty package path")
}
@ -835,11 +849,34 @@ func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd
buildMode = build.ImportComment
}
data.p, data.err = cfg.BuildContext.ImportDir(r.dir, buildMode)
if data.p.Root == "" && cfg.ModulesEnabled {
if info := modload.PackageModuleInfo(path); info != nil {
if cfg.ModulesEnabled {
// Override data.p.Root, since ImportDir sets it to $GOPATH, if
// the module is inside $GOPATH/src.
if info := modload.PackageModuleInfo(ctx, path); info != nil {
data.p.Root = info.Dir
}
}
if r.err != nil {
if data.err != nil {
// ImportDir gave us one error, and the module loader gave us another.
// We arbitrarily choose to keep the error from ImportDir because
// that's what our tests already expect, and it seems to provide a bit
// more detail in most cases.
} else if errors.Is(r.err, imports.ErrNoGo) {
// ImportDir said there were files in the package, but the module
// loader said there weren't. Which one is right?
// Without this special-case hack, the TestScript/test_vet case fails
// on the vetfail/p1 package (added in CL 83955).
// Apparently, imports.ShouldBuild biases toward rejecting files
// with invalid build constraints, whereas ImportDir biases toward
// accepting them.
//
// TODO(#41410: Figure out how this actually ought to work and fix
// this mess.
} else {
data.err = r.err
}
}
} else if r.err != nil {
data.p = new(build.Package)
data.err = r.err
@ -950,7 +987,7 @@ func newPreload() *preload {
// preloadMatches loads data for package paths matched by patterns.
// When preloadMatches returns, some packages may not be loaded yet, but
// loadPackageData and loadImport are always safe to call.
func (pre *preload) preloadMatches(matches []*search.Match) {
func (pre *preload) preloadMatches(ctx context.Context, opts PackageOpts, matches []*search.Match) {
for _, m := range matches {
for _, pkg := range m.Pkgs {
select {
@ -959,10 +996,10 @@ func (pre *preload) preloadMatches(matches []*search.Match) {
case pre.sema <- struct{}{}:
go func(pkg string) {
mode := 0 // don't use vendoring or module import resolution
bp, loaded, err := loadPackageData(pkg, "", base.Cwd, "", false, mode)
bp, loaded, err := loadPackageData(ctx, pkg, "", base.Cwd(), "", false, mode)
<-pre.sema
if bp != nil && loaded && err == nil && !IgnoreImports {
pre.preloadImports(bp.Imports, bp)
if bp != nil && loaded && err == nil && !opts.IgnoreImports {
pre.preloadImports(ctx, opts, bp.Imports, bp)
}
}(pkg)
}
@ -973,7 +1010,7 @@ func (pre *preload) preloadMatches(matches []*search.Match) {
// preloadImports queues a list of imports for preloading.
// When preloadImports returns, some packages may not be loaded yet,
// but loadPackageData and loadImport are always safe to call.
func (pre *preload) preloadImports(imports []string, parent *build.Package) {
func (pre *preload) preloadImports(ctx context.Context, opts PackageOpts, imports []string, parent *build.Package) {
parentIsStd := parent.Goroot && parent.ImportPath != "" && search.IsStandardImportPath(parent.ImportPath)
for _, path := range imports {
if path == "C" || path == "unsafe" {
@ -984,10 +1021,10 @@ func (pre *preload) preloadImports(imports []string, parent *build.Package) {
return
case pre.sema <- struct{}{}:
go func(path string) {
bp, loaded, err := loadPackageData(path, parent.ImportPath, parent.Dir, parent.Root, parentIsStd, ResolveImport)
bp, loaded, err := loadPackageData(ctx, path, parent.ImportPath, parent.Dir, parent.Root, parentIsStd, ResolveImport)
<-pre.sema
if bp != nil && loaded && err == nil && !IgnoreImports {
pre.preloadImports(bp.Imports, bp)
if bp != nil && loaded && err == nil && !opts.IgnoreImports {
pre.preloadImports(ctx, opts, bp.Imports, bp)
}
}(path)
}
@ -1323,6 +1360,11 @@ func reusePackage(p *Package, stk *ImportStack) *Package {
Err: errors.New("import cycle not allowed"),
IsImportCycle: true,
}
} else if !p.Error.IsImportCycle {
// If the error is already set, but it does not indicate that
// we are in an import cycle, set IsImportCycle so that we don't
// end up stuck in a loop down the road.
p.Error.IsImportCycle = true
}
p.Incomplete = true
}
@ -1338,7 +1380,7 @@ func reusePackage(p *Package, stk *ImportStack) *Package {
// is allowed to import p.
// If the import is allowed, disallowInternal returns the original package p.
// If not, it returns a new package containing just an appropriate error.
func disallowInternal(srcDir string, importer *Package, importerPath string, p *Package, stk *ImportStack) *Package {
func disallowInternal(ctx context.Context, srcDir string, importer *Package, importerPath string, p *Package, stk *ImportStack) *Package {
// golang.org/s/go14internal:
// An import of a path containing the element “internal”
// is disallowed if the importing code is outside the tree
@ -1416,7 +1458,7 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p *
// directory containing them.
// If the directory is outside the main module, this will resolve to ".",
// which is not a prefix of any valid module.
importerPath = modload.DirImportPath(importer.Dir)
importerPath = modload.DirImportPath(ctx, importer.Dir)
}
parentOfInternal := p.ImportPath[:i]
if str.HasPathPrefix(importerPath, parentOfInternal) {
@ -1638,8 +1680,8 @@ func (p *Package) DefaultExecName() string {
// load populates p using information from bp, err, which should
// be the result of calling build.Context.Import.
// stk contains the import stack, not including path itself.
func (p *Package) load(ctx context.Context, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) {
p.copyBuild(bp)
func (p *Package) load(ctx context.Context, opts PackageOpts, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) {
p.copyBuild(opts, bp)
// The localPrefix is the path we interpret ./ imports relative to.
// Synthesized main packages sometimes override this.
@ -1763,35 +1805,37 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor
}
}
// Cgo translation adds imports of "unsafe", "runtime/cgo" and "syscall",
// except for certain packages, to avoid circular dependencies.
if p.UsesCgo() {
addImport("unsafe", true)
}
if p.UsesCgo() && (!p.Standard || !cgoExclude[p.ImportPath]) && cfg.BuildContext.Compiler != "gccgo" {
addImport("runtime/cgo", true)
}
if p.UsesCgo() && (!p.Standard || !cgoSyscallExclude[p.ImportPath]) {
addImport("syscall", true)
}
// SWIG adds imports of some standard packages.
if p.UsesSwig() {
addImport("unsafe", true)
if cfg.BuildContext.Compiler != "gccgo" {
if !opts.IgnoreImports {
// Cgo translation adds imports of "unsafe", "runtime/cgo" and "syscall",
// except for certain packages, to avoid circular dependencies.
if p.UsesCgo() {
addImport("unsafe", true)
}
if p.UsesCgo() && (!p.Standard || !cgoExclude[p.ImportPath]) && cfg.BuildContext.Compiler != "gccgo" {
addImport("runtime/cgo", true)
}
addImport("syscall", true)
addImport("sync", true)
if p.UsesCgo() && (!p.Standard || !cgoSyscallExclude[p.ImportPath]) {
addImport("syscall", true)
}
// TODO: The .swig and .swigcxx files can use
// %go_import directives to import other packages.
}
// SWIG adds imports of some standard packages.
if p.UsesSwig() {
addImport("unsafe", true)
if cfg.BuildContext.Compiler != "gccgo" {
addImport("runtime/cgo", true)
}
addImport("syscall", true)
addImport("sync", true)
// The linker loads implicit dependencies.
if p.Name == "main" && !p.Internal.ForceLibrary {
for _, dep := range LinkerDeps(p) {
addImport(dep, false)
// TODO: The .swig and .swigcxx files can use
// %go_import directives to import other packages.
}
// The linker loads implicit dependencies.
if p.Name == "main" && !p.Internal.ForceLibrary {
for _, dep := range LinkerDeps(p) {
addImport(dep, false)
}
}
}
@ -1815,6 +1859,14 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor
stk.Push(path)
defer stk.Pop()
pkgPath := p.ImportPath
if p.Internal.CmdlineFiles {
pkgPath = "command-line-arguments"
}
if cfg.ModulesEnabled {
p.Module = modload.PackageModuleInfo(ctx, pkgPath)
}
p.EmbedFiles, p.Internal.Embed, err = resolveEmbed(p.Dir, p.EmbedPatterns)
if err != nil {
p.Incomplete = true
@ -1858,7 +1910,7 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor
if path == "C" {
continue
}
p1 := LoadImport(ctx, path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport)
p1 := LoadImport(ctx, opts, path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport)
path = p1.ImportPath
importPaths[i] = path
@ -1874,6 +1926,10 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor
p.Internal.Imports = imports
p.collectDeps()
if cfg.ModulesEnabled && p.Error == nil && p.Name == "main" && len(p.DepsErrors) == 0 {
p.Internal.BuildInfo = modload.PackageBuildInfo(pkgPath, p.Deps)
}
// unsafe is a fake package.
if p.Standard && (p.ImportPath == "unsafe" || cfg.BuildContext.Compiler == "gccgo") {
p.Target = ""
@ -1913,17 +1969,6 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor
setError(fmt.Errorf("Fortran source files not allowed when not using cgo or SWIG: %s", strings.Join(p.FFiles, " ")))
return
}
if cfg.ModulesEnabled && p.Error == nil {
mainPath := p.ImportPath
if p.Internal.CmdlineFiles {
mainPath = "command-line-arguments"
}
p.Module = modload.PackageModuleInfo(mainPath)
if p.Name == "main" && len(p.DepsErrors) == 0 {
p.Internal.BuildInfo = modload.PackageBuildInfo(mainPath, p.Deps)
}
}
}
// An EmbedError indicates a problem with a go:embed directive.
@ -2305,7 +2350,7 @@ func PackageList(roots []*Package) []*Package {
// TestPackageList returns the list of packages in the dag rooted at roots
// as visited in a depth-first post-order traversal, including the test
// imports of the roots. This ignores errors in test packages.
func TestPackageList(ctx context.Context, roots []*Package) []*Package {
func TestPackageList(ctx context.Context, opts PackageOpts, roots []*Package) []*Package {
seen := map[*Package]bool{}
all := []*Package{}
var walk func(*Package)
@ -2321,7 +2366,7 @@ func TestPackageList(ctx context.Context, roots []*Package) []*Package {
}
walkTest := func(root *Package, path string) {
var stk ImportStack
p1 := LoadImport(ctx, path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport)
p1 := LoadImport(ctx, opts, path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport)
if p1.Error == nil {
walk(p1)
}
@ -2344,22 +2389,35 @@ func TestPackageList(ctx context.Context, roots []*Package) []*Package {
// TODO(jayconrod): delete this function and set flags automatically
// in LoadImport instead.
func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
p := LoadImport(context.TODO(), path, srcDir, parent, stk, importPos, mode)
p := LoadImport(context.TODO(), PackageOpts{}, path, srcDir, parent, stk, importPos, mode)
setToolFlags(p)
return p
}
// ModResolveTests indicates whether calls to the module loader should also
// resolve test dependencies of the requested packages.
//
// If ModResolveTests is true, then the module loader needs to resolve test
// dependencies at the same time as packages; otherwise, the test dependencies
// of those packages could be missing, and resolving those missing dependencies
// could change the selected versions of modules that provide other packages.
//
// TODO(#40775): Change this from a global variable to an explicit function
// argument where needed.
var ModResolveTests bool
// PackageOpts control the behavior of PackagesAndErrors and other package
// loading functions.
type PackageOpts struct {
// IgnoreImports controls whether we ignore explicit and implicit imports
// when loading packages. Implicit imports are added when supporting Cgo
// or SWIG and when linking main packages.
IgnoreImports bool
// ModResolveTests indicates whether calls to the module loader should also
// resolve test dependencies of the requested packages.
//
// If ModResolveTests is true, then the module loader needs to resolve test
// dependencies at the same time as packages; otherwise, the test dependencies
// of those packages could be missing, and resolving those missing dependencies
// could change the selected versions of modules that provide other packages.
ModResolveTests bool
// MainOnly is true if the caller only wants to load main packages.
// For a literal argument matching a non-main package, a stub may be returned
// with an error. For a non-literal argument (with "..."), non-main packages
// are not be matched, and their dependencies may not be loaded. A warning
// may be printed for non-literal arguments that match no main packages.
MainOnly bool
}
// PackagesAndErrors returns the packages named by the command line arguments
// 'patterns'. If a named package cannot be loaded, PackagesAndErrors returns
@ -2369,7 +2427,7 @@ var ModResolveTests bool
//
// To obtain a flat list of packages, use PackageList.
// To report errors loading packages, use ReportPackageErrors.
func PackagesAndErrors(ctx context.Context, patterns []string) []*Package {
func PackagesAndErrors(ctx context.Context, opts PackageOpts, patterns []string) []*Package {
ctx, span := trace.StartSpan(ctx, "load.PackagesAndErrors")
defer span.Done()
@ -2381,19 +2439,19 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package {
// We need to test whether the path is an actual Go file and not a
// package path or pattern ending in '.go' (see golang.org/issue/34653).
if fi, err := fsys.Stat(p); err == nil && !fi.IsDir() {
return []*Package{GoFilesPackage(ctx, patterns)}
return []*Package{GoFilesPackage(ctx, opts, patterns)}
}
}
}
var matches []*search.Match
if modload.Init(); cfg.ModulesEnabled {
loadOpts := modload.PackageOpts{
modOpts := modload.PackageOpts{
ResolveMissingImports: true,
LoadTests: ModResolveTests,
SilenceErrors: true,
LoadTests: opts.ModResolveTests,
SilencePackageErrors: true,
}
matches, _ = modload.LoadPackages(ctx, loadOpts, patterns...)
matches, _ = modload.LoadPackages(ctx, modOpts, patterns...)
} else {
matches = search.ImportPaths(patterns)
}
@ -2406,14 +2464,14 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package {
pre := newPreload()
defer pre.flush()
pre.preloadMatches(matches)
pre.preloadMatches(ctx, opts, matches)
for _, m := range matches {
for _, pkg := range m.Pkgs {
if pkg == "" {
panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern()))
}
p := loadImport(ctx, pre, pkg, base.Cwd, nil, &stk, nil, 0)
p := loadImport(ctx, opts, pre, pkg, base.Cwd(), nil, &stk, nil, 0)
p.Match = append(p.Match, m.Pattern())
p.Internal.CmdlinePkg = true
if m.IsLiteral() {
@ -2449,6 +2507,10 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package {
}
}
if opts.MainOnly {
pkgs = mainPackagesOnly(pkgs, matches)
}
// Now that CmdlinePkg is set correctly,
// compute the effective flags for all loaded packages
// (not just the ones matching the patterns but also
@ -2497,6 +2559,80 @@ func CheckPackageErrors(pkgs []*Package) {
base.ExitIfErrors()
}
// mainPackagesOnly filters out non-main packages matched only by arguments
// containing "..." and returns the remaining main packages.
//
// Packages with missing, invalid, or ambiguous names may be treated as
// possibly-main packages.
//
// mainPackagesOnly sets a non-main package's Error field and returns it if it
// is named by a literal argument.
//
// mainPackagesOnly prints warnings for non-literal arguments that only match
// non-main packages.
func mainPackagesOnly(pkgs []*Package, matches []*search.Match) []*Package {
treatAsMain := map[string]bool{}
for _, m := range matches {
if m.IsLiteral() {
for _, path := range m.Pkgs {
treatAsMain[path] = true
}
}
}
var mains []*Package
for _, pkg := range pkgs {
if pkg.Name == "main" {
treatAsMain[pkg.ImportPath] = true
mains = append(mains, pkg)
continue
}
if len(pkg.InvalidGoFiles) > 0 { // TODO(#45999): && pkg.Name == "", but currently go/build sets pkg.Name arbitrarily if it is ambiguous.
// The package has (or may have) conflicting names, and we can't easily
// tell whether one of them is "main". So assume that it could be, and
// report an error for the package.
treatAsMain[pkg.ImportPath] = true
}
if treatAsMain[pkg.ImportPath] {
if pkg.Error == nil {
pkg.Error = &PackageError{Err: &mainPackageError{importPath: pkg.ImportPath}}
}
mains = append(mains, pkg)
}
}
for _, m := range matches {
if m.IsLiteral() || len(m.Pkgs) == 0 {
continue
}
foundMain := false
for _, path := range m.Pkgs {
if treatAsMain[path] {
foundMain = true
break
}
}
if !foundMain {
fmt.Fprintf(os.Stderr, "go: warning: %q matched only non-main packages\n", m.Pattern())
}
}
return mains
}
type mainPackageError struct {
importPath string
}
func (e *mainPackageError) Error() string {
return fmt.Sprintf("package %s is not a main package", e.importPath)
}
func (e *mainPackageError) ImportPath() string {
return e.importPath
}
func setToolFlags(pkgs ...*Package) {
for _, p := range PackageList(pkgs) {
p.Internal.Asmflags = BuildAsmflags.For(p)
@ -2509,7 +2645,7 @@ func setToolFlags(pkgs ...*Package) {
// GoFilesPackage creates a package for building a collection of Go files
// (typically named on the command line). The target is named p.a for
// package p or named after the first Go file for package main.
func GoFilesPackage(ctx context.Context, gofiles []string) *Package {
func GoFilesPackage(ctx context.Context, opts PackageOpts, gofiles []string) *Package {
modload.Init()
for _, f := range gofiles {
@ -2562,7 +2698,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package {
var err error
if dir == "" {
dir = base.Cwd
dir = base.Cwd()
}
dir, err = filepath.Abs(dir)
if err != nil {
@ -2573,7 +2709,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package {
pkg := new(Package)
pkg.Internal.Local = true
pkg.Internal.CmdlineFiles = true
pkg.load(ctx, "command-line-arguments", &stk, nil, bp, err)
pkg.load(ctx, opts, "command-line-arguments", &stk, nil, bp, err)
pkg.Internal.LocalPrefix = dirToImportPath(dir)
pkg.ImportPath = "command-line-arguments"
pkg.Target = ""
@ -2589,7 +2725,138 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package {
}
}
if opts.MainOnly && pkg.Name != "main" && pkg.Error == nil {
pkg.Error = &PackageError{Err: &mainPackageError{importPath: pkg.ImportPath}}
}
setToolFlags(pkg)
return pkg
}
// PackagesAndErrorsOutsideModule is like PackagesAndErrors but runs in
// module-aware mode and ignores the go.mod file in the current directory or any
// parent directory, if there is one. This is used in the implementation of 'go
// install pkg@version' and other commands that support similar forms.
//
// modload.ForceUseModules must be true, and modload.RootMode must be NoRoot
// before calling this function.
//
// PackagesAndErrorsOutsideModule imposes several constraints to avoid
// ambiguity. All arguments must have the same version suffix (not just a suffix
// that resolves to the same version). They must refer to packages in the same
// module, which must not be std or cmd. That module is not considered the main
// module, but its go.mod file (if it has one) must not contain directives that
// would cause it to be interpreted differently if it were the main module
// (replace, exclude).
func PackagesAndErrorsOutsideModule(ctx context.Context, opts PackageOpts, args []string) ([]*Package, error) {
if !modload.ForceUseModules {
panic("modload.ForceUseModules must be true")
}
if modload.RootMode != modload.NoRoot {
panic("modload.RootMode must be NoRoot")
}
// Check that the arguments satisfy syntactic constraints.
var version string
for _, arg := range args {
if i := strings.Index(arg, "@"); i >= 0 {
version = arg[i+1:]
if version == "" {
return nil, fmt.Errorf("%s: version must not be empty", arg)
}
break
}
}
patterns := make([]string, len(args))
for i, arg := range args {
if !strings.HasSuffix(arg, "@"+version) {
return nil, fmt.Errorf("%s: all arguments must have the same version (@%s)", arg, version)
}
p := arg[:len(arg)-len(version)-1]
switch {
case build.IsLocalImport(p):
return nil, fmt.Errorf("%s: argument must be a package path, not a relative path", arg)
case filepath.IsAbs(p):
return nil, fmt.Errorf("%s: argument must be a package path, not an absolute path", arg)
case search.IsMetaPackage(p):
return nil, fmt.Errorf("%s: argument must be a package path, not a meta-package", arg)
case path.Clean(p) != p:
return nil, fmt.Errorf("%s: argument must be a clean package path", arg)
case !strings.Contains(p, "...") && search.IsStandardImportPath(p) && goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, p):
return nil, fmt.Errorf("%s: argument must not be a package in the standard library", arg)
default:
patterns[i] = p
}
}
// Query the module providing the first argument, load its go.mod file, and
// check that it doesn't contain directives that would cause it to be
// interpreted differently if it were the main module.
//
// If multiple modules match the first argument, accept the longest match
// (first result). It's possible this module won't provide packages named by
// later arguments, and other modules would. Let's not try to be too
// magical though.
allowed := modload.CheckAllowed
if modload.IsRevisionQuery(version) {
// Don't check for retractions if a specific revision is requested.
allowed = nil
}
noneSelected := func(path string) (version string) { return "none" }
qrs, err := modload.QueryPackages(ctx, patterns[0], version, noneSelected, allowed)
if err != nil {
return nil, fmt.Errorf("%s: %w", args[0], err)
}
rootMod := qrs[0].Mod
data, err := modfetch.GoMod(rootMod.Path, rootMod.Version)
if err != nil {
return nil, fmt.Errorf("%s: %w", args[0], err)
}
f, err := modfile.Parse("go.mod", data, nil)
if err != nil {
return nil, fmt.Errorf("%s (in %s): %w", args[0], rootMod, err)
}
directiveFmt := "%s (in %s):\n" +
"\tThe go.mod file for the module providing named packages contains one or\n" +
"\tmore %s directives. It must not contain directives that would cause\n" +
"\tit to be interpreted differently than if it were the main module."
if len(f.Replace) > 0 {
return nil, fmt.Errorf(directiveFmt, args[0], rootMod, "replace")
}
if len(f.Exclude) > 0 {
return nil, fmt.Errorf(directiveFmt, args[0], rootMod, "exclude")
}
// Since we are in NoRoot mode, the build list initially contains only
// the dummy command-line-arguments module. Add a requirement on the
// module that provides the packages named on the command line.
if _, err := modload.EditBuildList(ctx, nil, []module.Version{rootMod}); err != nil {
return nil, fmt.Errorf("%s: %w", args[0], err)
}
// Load packages for all arguments.
pkgs := PackagesAndErrors(ctx, opts, patterns)
// Check that named packages are all provided by the same module.
for _, pkg := range pkgs {
var pkgErr error
if pkg.Module == nil {
// Packages in std, cmd, and their vendored dependencies
// don't have this field set.
pkgErr = fmt.Errorf("package %s not provided by module %s", pkg.ImportPath, rootMod)
} else if pkg.Module.Path != rootMod.Path || pkg.Module.Version != rootMod.Version {
pkgErr = fmt.Errorf("package %s provided by module %s@%s\n\tAll packages must be provided by the same module (%s).", pkg.ImportPath, pkg.Module.Path, pkg.Module.Version, rootMod)
}
if pkgErr != nil && pkg.Error == nil {
pkg.Error = &PackageError{Err: pkgErr}
}
}
matchers := make([]func(string) bool, len(patterns))
for i, p := range patterns {
if strings.Contains(p, "...") {
matchers[i] = search.MatchPattern(p)
}
}
return pkgs, nil
}

106
libgo/go/cmd/go/internal/load/test.go
View file

@ -21,6 +21,7 @@ import (
"unicode"
"unicode/utf8"
"cmd/go/internal/fsys"
"cmd/go/internal/str"
"cmd/go/internal/trace"
)
@ -45,8 +46,8 @@ type TestCover struct {
// TestPackagesFor is like TestPackagesAndErrors but it returns
// an error if the test packages or their dependencies have errors.
// Only test packages without errors are returned.
func TestPackagesFor(ctx context.Context, p *Package, cover *TestCover) (pmain, ptest, pxtest *Package, err error) {
pmain, ptest, pxtest = TestPackagesAndErrors(ctx, p, cover)
func TestPackagesFor(ctx context.Context, opts PackageOpts, p *Package, cover *TestCover) (pmain, ptest, pxtest *Package, err error) {
pmain, ptest, pxtest = TestPackagesAndErrors(ctx, opts, p, cover)
for _, p1 := range []*Package{ptest, pxtest, pmain} {
if p1 == nil {
// pxtest may be nil
@ -92,7 +93,7 @@ func TestPackagesFor(ctx context.Context, p *Package, cover *TestCover) (pmain,
//
// The caller is expected to have checked that len(p.TestGoFiles)+len(p.XTestGoFiles) > 0,
// or else there's no point in any of this.
func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (pmain, ptest, pxtest *Package) {
func TestPackagesAndErrors(ctx context.Context, opts PackageOpts, p *Package, cover *TestCover) (pmain, ptest, pxtest *Package) {
ctx, span := trace.StartSpan(ctx, "load.TestPackagesAndErrors")
defer span.Done()
@ -100,7 +101,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
defer pre.flush()
allImports := append([]string{}, p.TestImports...)
allImports = append(allImports, p.XTestImports...)
pre.preloadImports(allImports, p.Internal.Build)
pre.preloadImports(ctx, opts, allImports, p.Internal.Build)
var ptestErr, pxtestErr *PackageError
var imports, ximports []*Package
@ -109,13 +110,13 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
stk.Push(p.ImportPath + " (test)")
rawTestImports := str.StringList(p.TestImports)
for i, path := range p.TestImports {
p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport)
p1 := loadImport(ctx, opts, pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport)
if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath {
// Same error that loadPackage returns (via reusePackage) in pkg.go.
// Can't change that code, because that code is only for loading the
// non-test copy of a package.
ptestErr = &PackageError{
ImportStack: testImportStack(stk[0], p1, p.ImportPath),
ImportStack: importCycleStack(p1, p.ImportPath),
Err: errors.New("import cycle not allowed in test"),
IsImportCycle: true,
}
@ -139,7 +140,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
pxtestNeedsPtest := false
rawXTestImports := str.StringList(p.XTestImports)
for i, path := range p.XTestImports {
p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport)
p1 := loadImport(ctx, opts, pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport)
if p1.ImportPath == p.ImportPath {
pxtestNeedsPtest = true
} else {
@ -203,6 +204,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
}
ptest.Internal.Embed = testEmbed
ptest.EmbedFiles = str.StringList(p.EmbedFiles, p.TestEmbedFiles)
ptest.Internal.OrigImportPath = p.Internal.OrigImportPath
ptest.collectDeps()
} else {
ptest = p
@ -232,11 +234,12 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
Imports: ximports,
RawImports: rawXTestImports,
Asmflags: p.Internal.Asmflags,
Gcflags: p.Internal.Gcflags,
Ldflags: p.Internal.Ldflags,
Gccgoflags: p.Internal.Gccgoflags,
Embed: xtestEmbed,
Asmflags: p.Internal.Asmflags,
Gcflags: p.Internal.Gcflags,
Ldflags: p.Internal.Ldflags,
Gccgoflags: p.Internal.Gccgoflags,
Embed: xtestEmbed,
OrigImportPath: p.Internal.OrigImportPath,
},
}
if pxtestNeedsPtest {
@ -257,12 +260,13 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
Module: p.Module,
},
Internal: PackageInternal{
Build: &build.Package{Name: "main"},
BuildInfo: p.Internal.BuildInfo,
Asmflags: p.Internal.Asmflags,
Gcflags: p.Internal.Gcflags,
Ldflags: p.Internal.Ldflags,
Gccgoflags: p.Internal.Gccgoflags,
Build: &build.Package{Name: "main"},
BuildInfo: p.Internal.BuildInfo,
Asmflags: p.Internal.Asmflags,
Gcflags: p.Internal.Gcflags,
Ldflags: p.Internal.Ldflags,
Gccgoflags: p.Internal.Gccgoflags,
OrigImportPath: p.Internal.OrigImportPath,
},
}
@ -277,7 +281,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
if dep == ptest.ImportPath {
pmain.Internal.Imports = append(pmain.Internal.Imports, ptest)
} else {
p1 := loadImport(ctx, pre, dep, "", nil, &stk, nil, 0)
p1 := loadImport(ctx, opts, pre, dep, "", nil, &stk, nil, 0)
pmain.Internal.Imports = append(pmain.Internal.Imports, p1)
}
}
@ -290,10 +294,12 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
seen[p1] = true
}
for _, p1 := range cover.Pkgs {
if !seen[p1] {
seen[p1] = true
pmain.Internal.Imports = append(pmain.Internal.Imports, p1)
if seen[p1] {
// Don't add duplicate imports.
continue
}
seen[p1] = true
pmain.Internal.Imports = append(pmain.Internal.Imports, p1)
}
}
@ -369,22 +375,44 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
return pmain, ptest, pxtest
}
func testImportStack(top string, p *Package, target string) []string {
stk := []string{top, p.ImportPath}
Search:
for p.ImportPath != target {
for _, p1 := range p.Internal.Imports {
if p1.ImportPath == target || str.Contains(p1.Deps, target) {
stk = append(stk, p1.ImportPath)
p = p1
continue Search
// importCycleStack returns an import stack from p to the package whose import
// path is target.
func importCycleStack(p *Package, target string) []string {
// importerOf maps each import path to its importer nearest to p.
importerOf := map[string]string{p.ImportPath: ""}
// q is a breadth-first queue of packages to search for target.
// Every package added to q has a corresponding entry in pathTo.
//
// We search breadth-first for two reasons:
//
// 1. We want to report the shortest cycle.
//
// 2. If p contains multiple cycles, the first cycle we encounter might not
// contain target. To ensure termination, we have to break all cycles
// other than the first.
q := []*Package{p}
for len(q) > 0 {
p := q[0]
q = q[1:]
if path := p.ImportPath; path == target {
var stk []string
for path != "" {
stk = append(stk, path)
path = importerOf[path]
}
return stk
}
for _, dep := range p.Internal.Imports {
if _, ok := importerOf[dep.ImportPath]; !ok {
importerOf[dep.ImportPath] = p.ImportPath
q = append(q, dep)
}
}
// Can't happen, but in case it does...
stk = append(stk, "<lost path to cycle>")
break
}
return stk
panic("lost path to cycle")
}
// recompileForTest copies and replaces certain packages in pmain's dependency
@ -576,7 +604,13 @@ type testFunc struct {
var testFileSet = token.NewFileSet()
func (t *testFuncs) load(filename, pkg string, doImport, seen *bool) error {
f, err := parser.ParseFile(testFileSet, filename, nil, parser.ParseComments)
// Pass in the overlaid source if we have an overlay for this file.
src, err := fsys.Open(filename)
if err != nil {
return err
}
defer src.Close()
f, err := parser.ParseFile(testFileSet, filename, src, parser.ParseComments)
if err != nil {
return err
}

1
libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_fcntl.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build aix || (solaris && !illumos)
// +build aix solaris,!illumos
// This code implements the filelock API using POSIX 'fcntl' locks, which attach

1
libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_other.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !aix && !darwin && !dragonfly && !freebsd && !hurd && !linux && !netbsd && !openbsd && !plan9 && !solaris && !windows
// +build !aix,!darwin,!dragonfly,!freebsd,!hurd,!linux,!netbsd,!openbsd,!plan9,!solaris,!windows
package filelock

1
libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_plan9.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build plan9
// +build plan9
package filelock

1
libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_test.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !js && !plan9
// +build !js,!plan9
package filelock_test

1
libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_unix.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build darwin || dragonfly || freebsd || hurd || illumos || linux || netbsd || openbsd
// +build darwin dragonfly freebsd hurd illumos linux netbsd openbsd
package filelock

1
libgo/go/cmd/go/internal/lockedfile/internal/filelock/filelock_windows.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build windows
// +build windows
package filelock

4
libgo/go/cmd/go/internal/lockedfile/lockedfile_filelock.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !plan9
// +build !plan9
package lockedfile
@ -10,7 +11,6 @@ import (
"io/fs"
"os"
"cmd/go/internal/fsys"
"cmd/go/internal/lockedfile/internal/filelock"
)
@ -20,7 +20,7 @@ func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) {
// calls for Linux and Windows anyway, so it's simpler to use that approach
// consistently.
f, err := fsys.OpenFile(name, flag&^os.O_TRUNC, perm)
f, err := os.OpenFile(name, flag&^os.O_TRUNC, perm)
if err != nil {
return nil, err
}

7
libgo/go/cmd/go/internal/lockedfile/lockedfile_plan9.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build plan9
// +build plan9
package lockedfile
@ -12,8 +13,6 @@ import (
"os"
"strings"
"time"
"cmd/go/internal/fsys"
)
// Opening an exclusive-use file returns an error.
@ -58,7 +57,7 @@ func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) {
// If the file was unpacked or created by some other program, it might not
// have the ModeExclusive bit set. Set it before we call OpenFile, so that we
// can be confident that a successful OpenFile implies exclusive use.
if fi, err := fsys.Stat(name); err == nil {
if fi, err := os.Stat(name); err == nil {
if fi.Mode()&fs.ModeExclusive == 0 {
if err := os.Chmod(name, fi.Mode()|fs.ModeExclusive); err != nil {
return nil, err
@ -71,7 +70,7 @@ func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) {
nextSleep := 1 * time.Millisecond
const maxSleep = 500 * time.Millisecond
for {
f, err := fsys.OpenFile(name, flag, perm|fs.ModeExclusive)
f, err := os.OpenFile(name, flag, perm|fs.ModeExclusive)
if err == nil {
return f, nil
}

1
libgo/go/cmd/go/internal/lockedfile/lockedfile_test.go
View file

@ -3,6 +3,7 @@
// license that can be found in the LICENSE file.
// js does not support inter-process file locking.
//go:build !js
// +build !js
package lockedfile_test

1
libgo/go/cmd/go/internal/lockedfile/transform_test.go
View file

@ -3,6 +3,7 @@
// license that can be found in the LICENSE file.
// js does not support inter-process file locking.
//go:build !js
// +build !js
package lockedfile_test

30
libgo/go/cmd/go/internal/modcmd/download.go
View file

@ -134,21 +134,18 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
}
var mods []*moduleJSON
listU := false
listVersions := false
listRetractions := false
type token struct{}
sem := make(chan token, runtime.GOMAXPROCS(0))
infos := modload.ListModules(ctx, args, listU, listVersions, listRetractions)
infos, infosErr := modload.ListModules(ctx, args, 0)
if !haveExplicitArgs {
// 'go mod download' is sometimes run without arguments to pre-populate
// the module cache. It may fetch modules that aren't needed to build
// packages in the main mdoule. This is usually not intended, so don't save
// sums for downloaded modules (golang.org/issue/45332).
// TODO(golang.org/issue/45551): For now, save sums needed to load the
// build list (same as 1.15 behavior). In the future, report an error if
// go.mod or go.sum need to be updated after loading the build list.
modload.WriteGoMod()
// 'go mod download' is sometimes run without arguments to pre-populate the
// module cache. It may fetch modules that aren't needed to build packages
// in the main mdoule. This is usually not intended, so don't save sums for
// downloaded modules (golang.org/issue/45332).
// TODO(golang.org/issue/45551): For now, in ListModules, save sums needed
// to load the build list (same as 1.15 behavior). In the future, report an
// error if go.mod or go.sum need to be updated after loading the build
// list.
modload.DisallowWriteGoMod()
}
@ -209,6 +206,13 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
//
// Don't save sums for 'go mod download' without arguments; see comment above.
if haveExplicitArgs {
modload.WriteGoMod()
modload.WriteGoMod(ctx)
}
// If there was an error matching some of the requested packages, emit it now
// (after we've written the checksums for the modules that were downloaded
// successfully).
if infosErr != nil {
base.Errorf("go mod download: %v", infosErr)
}
}

25
libgo/go/cmd/go/internal/modcmd/edit.go
View file

@ -25,7 +25,7 @@ import (
)
var cmdEdit = &base.Command{
UsageLine: "go mod edit [editing flags] [go.mod]",
UsageLine: "go mod edit [editing flags] [-fmt|-print|-json] [go.mod]",
Short: "edit go.mod from tools or scripts",
Long: `
Edit provides a command-line interface for editing go.mod,
@ -85,12 +85,12 @@ The -json flag prints the final go.mod file in JSON format instead of
writing it back to go.mod. The JSON output corresponds to these Go types:
type Module struct {
Path string
Path string
Version string
}
type GoMod struct {
Module Module
Module ModPath
Go string
Require []Require
Exclude []Module
@ -98,6 +98,11 @@ writing it back to go.mod. The JSON output corresponds to these Go types:
Retract []Retract
}
type ModPath struct {
Path string
Deprecated string
}
type Require struct {
Path string
Version string
@ -191,7 +196,7 @@ func runEdit(ctx context.Context, cmd *base.Command, args []string) {
if *editGo != "" {
if !modfile.GoVersionRE.MatchString(*editGo) {
base.Fatalf(`go mod: invalid -go option; expecting something like "-go 1.12"`)
base.Fatalf(`go mod: invalid -go option; expecting something like "-go %s"`, modload.LatestGoVersion())
}
}
@ -450,7 +455,7 @@ func flagDropRetract(arg string) {
// fileJSON is the -json output data structure.
type fileJSON struct {
Module module.Version
Module editModuleJSON
Go string `json:",omitempty"`
Require []requireJSON
Exclude []module.Version
@ -458,6 +463,11 @@ type fileJSON struct {
Retract []retractJSON
}
type editModuleJSON struct {
Path string
Deprecated string `json:",omitempty"`
}
type requireJSON struct {
Path string
Version string `json:",omitempty"`
@ -479,7 +489,10 @@ type retractJSON struct {
func editPrintJSON(modFile *modfile.File) {
var f fileJSON
if modFile.Module != nil {
f.Module = modFile.Module.Mod
f.Module = editModuleJSON{
Path: modFile.Module.Mod.Path,
Deprecated: modFile.Module.Deprecated,
}
}
if modFile.Go != nil {
f.Go = modFile.Go.Version

67
libgo/go/cmd/go/internal/modcmd/graph.go
View file

@ -10,7 +10,6 @@ import (
"bufio"
"context"
"os"
"sort"
"cmd/go/internal/base"
"cmd/go/internal/modload"
@ -19,7 +18,7 @@ import (
)
var cmdGraph = &base.Command{
UsageLine: "go mod graph",
UsageLine: "go mod graph [-go=version]",
Short: "print module requirement graph",
Long: `
Graph prints the module requirement graph (with replacements applied)
@ -27,12 +26,21 @@ in text form. Each line in the output has two space-separated fields: a module
and one of its requirements. Each module is identified as a string of the form
path@version, except for the main module, which has no @version suffix.
The -go flag causes graph to report the module graph as loaded by the
given Go version, instead of the version indicated by the 'go' directive
in the go.mod file.
See https://golang.org/ref/mod#go-mod-graph for more about 'go mod graph'.
`,
Run: runGraph,
}
var (
graphGo goVersionFlag
)
func init() {
cmdGraph.Flag.Var(&graphGo, "go", "")
base.AddModCommonFlags(&cmdGraph.Flag)
}
@ -42,43 +50,26 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) {
}
modload.ForceUseModules = true
modload.RootMode = modload.NeedRoot
modload.LoadAllModules(ctx)
reqs := modload.MinReqs()
format := func(m module.Version) string {
if m.Version == "" {
return m.Path
}
return m.Path + "@" + m.Version
}
var out []string
var deps int // index in out where deps start
seen := map[module.Version]bool{modload.Target: true}
queue := []module.Version{modload.Target}
for len(queue) > 0 {
var m module.Version
m, queue = queue[0], queue[1:]
list, _ := reqs.Required(m)
for _, r := range list {
if !seen[r] {
queue = append(queue, r)
seen[r] = true
}
out = append(out, format(m)+" "+format(r)+"\n")
}
if m == modload.Target {
deps = len(out)
}
}
sort.Slice(out[deps:], func(i, j int) bool {
return out[deps+i][0] < out[deps+j][0]
})
mg := modload.LoadModGraph(ctx, graphGo.String())
w := bufio.NewWriter(os.Stdout)
for _, line := range out {
w.WriteString(line)
defer w.Flush()
format := func(m module.Version) {
w.WriteString(m.Path)
if m.Version != "" {
w.WriteString("@")
w.WriteString(m.Version)
}
}
w.Flush()
mg.WalkBreadthFirst(func(m module.Version) {
reqs, _ := mg.RequiredBy(m)
for _, r := range reqs {
format(m)
w.WriteByte(' ')
format(r)
w.WriteByte('\n')
}
})
}

2
libgo/go/cmd/go/internal/modcmd/init.go
View file

@ -13,7 +13,7 @@ import (
)
var cmdInit = &base.Command{
UsageLine: "go mod init [module]",
UsageLine: "go mod init [module-path]",
Short: "initialize new module in current directory",
Long: `
Init initializes and writes a new go.mod file in the current directory, in

64
libgo/go/cmd/go/internal/modcmd/tidy.go
View file

@ -12,10 +12,14 @@ import (
"cmd/go/internal/imports"
"cmd/go/internal/modload"
"context"
"fmt"
"golang.org/x/mod/modfile"
"golang.org/x/mod/semver"
)
var cmdTidy = &base.Command{
UsageLine: "go mod tidy [-e] [-v]",
UsageLine: "go mod tidy [-e] [-v] [-go=version] [-compat=version]",
Short: "add missing and remove unused modules",
Long: `
Tidy makes sure go.mod matches the source code in the module.
@ -30,19 +34,65 @@ to standard error.
The -e flag causes tidy to attempt to proceed despite errors
encountered while loading packages.
The -go flag causes tidy to update the 'go' directive in the go.mod
file to the given version, which may change which module dependencies
are retained as explicit requirements in the go.mod file.
(Go versions 1.17 and higher retain more requirements in order to
support lazy module loading.)
The -compat flag preserves any additional checksums needed for the
'go' command from the indicated major Go release to successfully load
the module graph, and causes tidy to error out if that version of the
'go' command would load any imported package from a different module
version. By default, tidy acts as if the -compat flag were set to the
version prior to the one indicated by the 'go' directive in the go.mod
file.
See https://golang.org/ref/mod#go-mod-tidy for more about 'go mod tidy'.
`,
Run: runTidy,
}
var tidyE bool // if true, report errors but proceed anyway.
var (
tidyE bool // if true, report errors but proceed anyway.
tidyGo goVersionFlag // go version to write to the tidied go.mod file (toggles lazy loading)
tidyCompat goVersionFlag // go version for which the tidied go.mod and go.sum files should be “compatible”
)
func init() {
cmdTidy.Flag.BoolVar(&cfg.BuildV, "v", false, "")
cmdTidy.Flag.BoolVar(&tidyE, "e", false, "")
cmdTidy.Flag.Var(&tidyGo, "go", "")
cmdTidy.Flag.Var(&tidyCompat, "compat", "")
base.AddModCommonFlags(&cmdTidy.Flag)
}
// A goVersionFlag is a flag.Value representing a supported Go version.
//
// (Note that the -go argument to 'go mod edit' is *not* a goVersionFlag.
// It intentionally allows newer-than-supported versions as arguments.)
type goVersionFlag struct {
v string
}
func (f *goVersionFlag) String() string { return f.v }
func (f *goVersionFlag) Get() interface{} { return f.v }
func (f *goVersionFlag) Set(s string) error {
if s != "" {
latest := modload.LatestGoVersion()
if !modfile.GoVersionRE.MatchString(s) {
return fmt.Errorf("expecting a Go version like %q", latest)
}
if semver.Compare("v"+s, "v"+latest) > 0 {
return fmt.Errorf("maximum supported Go version is %s", latest)
}
}
f.v = s
return nil
}
func runTidy(ctx context.Context, cmd *base.Command, args []string) {
if len(args) > 0 {
base.Fatalf("go mod tidy: no arguments allowed")
@ -61,17 +111,15 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) {
modload.ForceUseModules = true
modload.RootMode = modload.NeedRoot
modload.CheckTidyVersion(ctx, tidyE)
modload.LoadPackages(ctx, modload.PackageOpts{
GoVersion: tidyGo.String(),
Tags: imports.AnyTags(),
Tidy: true,
TidyCompatibleVersion: tidyCompat.String(),
VendorModulesInGOROOTSrc: true,
ResolveMissingImports: true,
LoadTests: true,
AllowErrors: tidyE,
SilenceMissingStdImports: true,
}, "all")
modload.TidyBuildList()
modload.TrimGoSum()
modload.WriteGoMod()
}

70
libgo/go/cmd/go/internal/modcmd/vendor.go
View file

@ -13,6 +13,7 @@ import (
"io"
"io/fs"
"os"
"path"
"path/filepath"
"sort"
"strings"
@ -65,6 +66,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) {
loadOpts := modload.PackageOpts{
Tags: imports.AnyTags(),
VendorModulesInGOROOTSrc: true,
ResolveMissingImports: true,
UseVendorAll: true,
AllowErrors: vendorE,
@ -87,15 +89,23 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) {
}
includeAllReplacements := false
includeGoVersions := false
isExplicit := map[module.Version]bool{}
if gv := modload.ModFile().Go; gv != nil && semver.Compare("v"+gv.Version, "v1.14") >= 0 {
// If the Go version is at least 1.14, annotate all explicit 'require' and
// 'replace' targets found in the go.mod file so that we can perform a
// stronger consistency check when -mod=vendor is set.
for _, r := range modload.ModFile().Require {
isExplicit[r.Mod] = true
if gv := modload.ModFile().Go; gv != nil {
if semver.Compare("v"+gv.Version, "v1.14") >= 0 {
// If the Go version is at least 1.14, annotate all explicit 'require' and
// 'replace' targets found in the go.mod file so that we can perform a
// stronger consistency check when -mod=vendor is set.
for _, r := range modload.ModFile().Require {
isExplicit[r.Mod] = true
}
includeAllReplacements = true
}
if semver.Compare("v"+gv.Version, "v1.17") >= 0 {
// If the Go version is at least 1.17, annotate all modules with their
// 'go' version directives.
includeGoVersions = true
}
includeAllReplacements = true
}
var vendorMods []module.Version
@ -109,26 +119,35 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) {
}
module.Sort(vendorMods)
var buf bytes.Buffer
var (
buf bytes.Buffer
w io.Writer = &buf
)
if cfg.BuildV {
w = io.MultiWriter(&buf, os.Stderr)
}
for _, m := range vendorMods {
line := moduleLine(m, modload.Replacement(m))
buf.WriteString(line)
if cfg.BuildV {
os.Stderr.WriteString(line)
io.WriteString(w, line)
goVersion := ""
if includeGoVersions {
goVersion = modload.ModuleInfo(ctx, m.Path).GoVersion
}
if isExplicit[m] {
buf.WriteString("## explicit\n")
if cfg.BuildV {
os.Stderr.WriteString("## explicit\n")
}
switch {
case isExplicit[m] && goVersion != "":
fmt.Fprintf(w, "## explicit; go %s\n", goVersion)
case isExplicit[m]:
io.WriteString(w, "## explicit\n")
case goVersion != "":
fmt.Fprintf(w, "## go %s\n", goVersion)
}
pkgs := modpkgs[m]
sort.Strings(pkgs)
for _, pkg := range pkgs {
fmt.Fprintf(&buf, "%s\n", pkg)
if cfg.BuildV {
fmt.Fprintf(os.Stderr, "%s\n", pkg)
}
fmt.Fprintf(w, "%s\n", pkg)
vendorPkg(vdir, pkg)
}
}
@ -281,7 +300,7 @@ func copyMetadata(modPath, pkg, dst, src string, copiedFiles map[string]bool) {
if modPath == pkg {
break
}
pkg = filepath.Dir(pkg)
pkg = path.Dir(pkg)
dst = filepath.Dir(dst)
src = filepath.Dir(src)
}
@ -322,6 +341,15 @@ func matchPotentialSourceFile(dir string, info fs.DirEntry) bool {
if strings.HasSuffix(info.Name(), "_test.go") {
return false
}
if info.Name() == "go.mod" || info.Name() == "go.sum" {
if gv := modload.ModFile().Go; gv != nil && semver.Compare("v"+gv.Version, "v1.17") >= 0 {
// As of Go 1.17, we strip go.mod and go.sum files from dependency modules.
// Otherwise, 'go' commands invoked within the vendor subtree may misidentify
// an arbitrary directory within the vendor tree as a module root.
// (See https://golang.org/issue/42970.)
return false
}
}
if strings.HasSuffix(info.Name(), ".go") {
f, err := fsys.Open(filepath.Join(dir, info.Name()))
if err != nil {

3
libgo/go/cmd/go/internal/modcmd/verify.go
View file

@ -54,7 +54,8 @@ func runVerify(ctx context.Context, cmd *base.Command, args []string) {
sem := make(chan token, runtime.GOMAXPROCS(0))
// Use a slice of result channels, so that the output is deterministic.
mods := modload.LoadAllModules(ctx)[1:]
const defaultGoVersion = ""
mods := modload.LoadModGraph(ctx, defaultGoVersion).BuildList()[1:]
errsChans := make([]<-chan []error, len(mods))
for i, mod := range mods {

19
libgo/go/cmd/go/internal/modcmd/why.go
View file

@ -68,22 +68,25 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) {
modload.RootMode = modload.NeedRoot
loadOpts := modload.PackageOpts{
Tags: imports.AnyTags(),
LoadTests: !*whyVendor,
SilenceErrors: true,
UseVendorAll: *whyVendor,
Tags: imports.AnyTags(),
VendorModulesInGOROOTSrc: true,
LoadTests: !*whyVendor,
SilencePackageErrors: true,
UseVendorAll: *whyVendor,
}
if *whyM {
listU := false
listVersions := false
listRetractions := false
for _, arg := range args {
if strings.Contains(arg, "@") {
base.Fatalf("go mod why: module query not allowed")
}
}
mods := modload.ListModules(ctx, args, listU, listVersions, listRetractions)
mods, err := modload.ListModules(ctx, args, 0)
if err != nil {
base.Fatalf("go mod why: %v", err)
}
byModule := make(map[module.Version][]string)
_, pkgs := modload.LoadPackages(ctx, loadOpts, "all")
for _, path := range pkgs {

9
libgo/go/cmd/go/internal/modconv/convert.go
View file

@ -12,7 +12,6 @@ import (
"strings"
"cmd/go/internal/base"
"cmd/go/internal/modfetch"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
@ -21,7 +20,7 @@ import (
// ConvertLegacyConfig converts legacy config to modfile.
// The file argument is slash-delimited.
func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error {
func ConvertLegacyConfig(f *modfile.File, file string, data []byte, queryPackage func(path, rev string) (module.Version, error)) error {
i := strings.LastIndex(file, "/")
j := -2
if i >= 0 {
@ -62,15 +61,13 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error {
sem <- token{}
go func(i int, m module.Version) {
defer func() { <-sem }()
repo, info, err := modfetch.ImportRepoRev(m.Path, m.Version)
version, err := queryPackage(m.Path, m.Version)
if err != nil {
fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), m.Path, m.Version, err)
return
}
path := repo.ModulePath()
versions[i].Path = path
versions[i].Version = info.Version
versions[i] = version
}(i, m)
}
// Fill semaphore channel to wait for all tasks to finish.

189
libgo/go/cmd/go/internal/modconv/convert_test.go
View file

@ -1,189 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package modconv
import (
"bytes"
"context"
"fmt"
"internal/testenv"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"cmd/go/internal/cfg"
"cmd/go/internal/modfetch"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
func TestMain(m *testing.M) {
os.Exit(testMain(m))
}
func testMain(m *testing.M) int {
cfg.GOPROXY = "direct"
if _, err := exec.LookPath("git"); err != nil {
fmt.Fprintln(os.Stderr, "skipping because git binary not found")
fmt.Println("PASS")
return 0
}
dir, err := os.MkdirTemp("", "modconv-test-")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(dir)
cfg.GOMODCACHE = filepath.Join(dir, "pkg/mod")
return m.Run()
}
func TestConvertLegacyConfig(t *testing.T) {
testenv.MustHaveExternalNetwork(t)
if testing.Verbose() {
old := cfg.BuildX
defer func() {
cfg.BuildX = old
}()
cfg.BuildX = true
}
var tests = []struct {
path string
vers string
gomod string
}{
/*
Different versions of git seem to find or not find
github.com/Masterminds/semver's a93e51b5a57e,
which is an unmerged pull request.
We'd rather not provide access to unmerged pull requests,
so the line is removed from the golden file here,
but some git commands still find it somehow.
{
// Gopkg.lock parsing.
"github.com/golang/dep", "v0.4.0",
`module github.com/golang/dep
require (
github.com/Masterminds/vcs v1.11.1
github.com/armon/go-radix v0.0.0-20160115234725-4239b77079c7
github.com/boltdb/bolt v1.3.1
github.com/go-yaml/yaml v0.0.0-20170407172122-cd8b52f8269e
github.com/golang/protobuf v0.0.0-20170901042739-5afd06f9d81a
github.com/jmank88/nuts v0.3.0
github.com/nightlyone/lockfile v0.0.0-20170707060451-e83dc5e7bba0
github.com/pelletier/go-toml v0.0.0-20171218135716-b8b5e7696574
github.com/pkg/errors v0.8.0
github.com/sdboyer/constext v0.0.0-20170321163424-836a14457353
golang.org/x/net v0.0.0-20170828231752-66aacef3dd8a
golang.org/x/sync v0.0.0-20170517211232-f52d1811a629
golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea
)`,
},
*/
// TODO: https://github.com/docker/distribution uses vendor.conf
{
// Godeps.json parsing.
// TODO: Should v2.0.0 work here too?
"github.com/docker/distribution", "v0.0.0-20150410205453-85de3967aa93",
`module github.com/docker/distribution
require (
github.com/AdRoll/goamz v0.0.0-20150130162828-d3664b76d905
github.com/MSOpenTech/azure-sdk-for-go v0.0.0-20150323223030-d90753bcad2e
github.com/Sirupsen/logrus v0.7.3
github.com/bugsnag/bugsnag-go v1.0.3-0.20141110184014-b1d153021fcd
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b
github.com/bugsnag/panicwrap v0.0.0-20141110184334-e5f9854865b9
github.com/codegangsta/cli v1.4.2-0.20150131031259-6086d7927ec3
github.com/docker/docker v1.4.2-0.20150204013315-165ea5c158cf
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1
github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7
github.com/gorilla/context v0.0.0-20140604161150-14f550f51af5
github.com/gorilla/handlers v0.0.0-20140825150757-0e84b7d810c1
github.com/gorilla/mux v0.0.0-20140926153814-e444e69cbd2e
github.com/jlhawn/go-crypto v0.0.0-20150401213827-cd738dde20f0
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43
github.com/yvasiyarov/gorelic v0.0.7-0.20141212073537-a9bba5b9ab50
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f
golang.org/x/net v0.0.0-20150202051010-1dfe7915deaf
gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789
gopkg.in/yaml.v2 v2.0.0-20150116202057-bef53efd0c76
)`,
},
{
// golang.org/issue/24585 - confusion about v2.0.0 tag in legacy non-v2 module
"github.com/fishy/gcsbucket", "v0.0.0-20180217031846-618d60fe84e0",
`module github.com/fishy/gcsbucket
require (
cloud.google.com/go v0.18.0
github.com/fishy/fsdb v0.0.0-20180217030800-5527ded01371
github.com/golang/protobuf v1.0.0
github.com/googleapis/gax-go v2.0.0+incompatible
golang.org/x/net v0.0.0-20180216171745-136a25c244d3
golang.org/x/oauth2 v0.0.0-20180207181906-543e37812f10
golang.org/x/text v0.3.1-0.20180208041248-4e4a3210bb54
google.golang.org/api v0.0.0-20180217000815-c7a403bb5fe1
google.golang.org/appengine v1.0.0
google.golang.org/genproto v0.0.0-20180206005123-2b5a72b8730b
google.golang.org/grpc v1.10.0
)`,
},
}
ctx := context.Background()
for _, tt := range tests {
t.Run(strings.ReplaceAll(tt.path, "/", "_")+"_"+tt.vers, func(t *testing.T) {
f, err := modfile.Parse("golden", []byte(tt.gomod), nil)
if err != nil {
t.Fatal(err)
}
want, err := f.Format()
if err != nil {
t.Fatal(err)
}
dir, err := modfetch.Download(ctx, module.Version{Path: tt.path, Version: tt.vers})
if err != nil {
t.Fatal(err)
}
for name := range Converters {
file := filepath.Join(dir, name)
data, err := os.ReadFile(file)
if err == nil {
f := new(modfile.File)
f.AddModuleStmt(tt.path)
if err := ConvertLegacyConfig(f, filepath.ToSlash(file), data); err != nil {
t.Fatal(err)
}
out, err := f.Format()
if err != nil {
t.Fatalf("format after conversion: %v", err)
}
if !bytes.Equal(out, want) {
t.Fatalf("final go.mod:\n%s\n\nwant:\n%s", out, want)
}
return
}
}
t.Fatalf("no converter found for %s@%s", tt.path, tt.vers)
})
}
}

79
libgo/go/cmd/go/internal/modconv/testdata/traefik.dep vendored Normal file
View file

@ -0,0 +1,79 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "github.com/Nvveen/Gotty"
packages = ["."]
revision = "a8b993ba6abdb0e0c12b0125c603323a71c7790c"
source = "github.com/ijc25/Gotty"
[[projects]]
branch = "master"
name = "github.com/OpenDNS/vegadns2client"
packages = ["."]
revision = "a3fa4a771d87bda2514a90a157e1fed1b6897d2e"
[[projects]]
name = "github.com/PuerkitoBio/purell"
packages = ["."]
revision = "8a290539e2e8629dbc4e6bad948158f790ec31f4"
version = "v1.0.0"
[[projects]]
name = "github.com/PuerkitoBio/urlesc"
packages = ["."]
revision = "5bd2802263f21d8788851d5305584c82a5c75d7e"
[[projects]]
name = "github.com/Shopify/sarama"
packages = ["."]
revision = "70f6a705d4a17af059acbc6946fb2bd30762acd7"
[[projects]]
name = "github.com/VividCortex/gohistogram"
packages = ["."]
revision = "51564d9861991fb0ad0f531c99ef602d0f9866e6"
version = "v1.0.0"
[[projects]]
branch = "containous-fork"
name = "github.com/abbot/go-http-auth"
packages = ["."]
revision = "65b0cdae8d7fe5c05c7430e055938ef6d24a66c9"
source = "github.com/containous/go-http-auth"
[[projects]]
branch = "master"
name = "github.com/abronan/valkeyrie"
packages = [
".",
"store",
"store/boltdb",
"store/consul",
"store/etcd/v2",
"store/etcd/v3",
"store/zookeeper"
]
revision = "063d875e3c5fd734fa2aa12fac83829f62acfc70"
[[projects]]
branch = "master"
name = "github.com/mesosphere/mesos-dns"
packages = [
"detect",
"errorutil",
"logging",
"models",
"records",
"records/labels",
"records/state",
"util"
]
revision = "b47dc4c19f215e98da687b15b4c64e70f629bea5"
source = "git@github.com:containous/mesos-dns.git"
[[projects]]
name = "gopkg.in/fsnotify.v1"
packages = ["."]
revision = "629574ca2a5df945712d3079857300b5e4da0236"
source = "github.com/fsnotify/fsnotify"
version = "v1.4.2"

14
libgo/go/cmd/go/internal/modconv/testdata/traefik.out vendored Normal file
View file

@ -0,0 +1,14 @@
github.com/Nvveen/Gotty a8b993ba6abdb0e0c12b0125c603323a71c7790c
github.com/OpenDNS/vegadns2client a3fa4a771d87bda2514a90a157e1fed1b6897d2e
github.com/PuerkitoBio/purell v1.0.0
github.com/PuerkitoBio/urlesc 5bd2802263f21d8788851d5305584c82a5c75d7e
github.com/Shopify/sarama 70f6a705d4a17af059acbc6946fb2bd30762acd7
github.com/VividCortex/gohistogram v1.0.0
github.com/abbot/go-http-auth 65b0cdae8d7fe5c05c7430e055938ef6d24a66c9
github.com/abronan/valkeyrie 063d875e3c5fd734fa2aa12fac83829f62acfc70
github.com/mesosphere/mesos-dns b47dc4c19f215e98da687b15b4c64e70f629bea5
gopkg.in/fsnotify.v1 v1.4.2
replace: github.com/Nvveen/Gotty a8b993ba6abdb0e0c12b0125c603323a71c7790c github.com/ijc25/Gotty a8b993ba6abdb0e0c12b0125c603323a71c7790c
replace: github.com/abbot/go-http-auth 65b0cdae8d7fe5c05c7430e055938ef6d24a66c9 github.com/containous/go-http-auth 65b0cdae8d7fe5c05c7430e055938ef6d24a66c9
replace: github.com/mesosphere/mesos-dns b47dc4c19f215e98da687b15b4c64e70f629bea5 github.com/containous/mesos-dns b47dc4c19f215e98da687b15b4c64e70f629bea5
replace: gopkg.in/fsnotify.v1 v1.4.2 github.com/fsnotify/fsnotify v1.4.2

1
libgo/go/cmd/go/internal/modfetch/bootstrap.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build cmd_go_bootstrap
// +build cmd_go_bootstrap
package modfetch

174
libgo/go/cmd/go/internal/modfetch/cache.go
View file

@ -11,8 +11,10 @@ import (
"fmt"
"io"
"io/fs"
"math/rand"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
@ -21,17 +23,15 @@ import (
"cmd/go/internal/lockedfile"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/par"
"cmd/go/internal/renameio"
"cmd/go/internal/robustio"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
)
func cacheDir(path string) (string, error) {
if cfg.GOMODCACHE == "" {
// modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE
// is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen.
return "", fmt.Errorf("internal error: cfg.GOMODCACHE not set")
if err := checkCacheDir(); err != nil {
return "", err
}
enc, err := module.EscapePath(path)
if err != nil {
@ -64,10 +64,8 @@ func CachePath(m module.Version, suffix string) (string, error) {
// along with the directory if the directory does not exist or if the directory
// is not completely populated.
func DownloadDir(m module.Version) (string, error) {
if cfg.GOMODCACHE == "" {
// modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE
// is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen.
return "", fmt.Errorf("internal error: cfg.GOMODCACHE not set")
if err := checkCacheDir(); err != nil {
return "", err
}
enc, err := module.EscapePath(m.Path)
if err != nil {
@ -108,7 +106,9 @@ func DownloadDir(m module.Version) (string, error) {
// Check if a .ziphash file exists. It should be created before the
// zip is extracted, but if it was deleted (by another program?), we need
// to re-calculate it.
// to re-calculate it. Note that checkMod will repopulate the ziphash
// file if it doesn't exist, but if the module is excluded by checks
// through GONOSUMDB or GOPRIVATE, that check and repopulation won't happen.
ziphashPath, err := CachePath(m, "ziphash")
if err != nil {
return dir, err
@ -146,15 +146,13 @@ func lockVersion(mod module.Version) (unlock func(), err error) {
return lockedfile.MutexAt(path).Lock()
}
// SideLock locks a file within the module cache that that previously guarded
// SideLock locks a file within the module cache that previously guarded
// edits to files outside the cache, such as go.sum and go.mod files in the
// user's working directory.
// If err is nil, the caller MUST eventually call the unlock function.
func SideLock() (unlock func(), err error) {
if cfg.GOMODCACHE == "" {
// modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE
// is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen.
base.Fatalf("go: internal error: cfg.GOMODCACHE not set")
if err := checkCacheDir(); err != nil {
return nil, err
}
path := filepath.Join(cfg.GOMODCACHE, "cache", "lock")
@ -332,7 +330,7 @@ func InfoFile(path, version string) (string, error) {
}
// Stat should have populated the disk cache for us.
file, _, err := readDiskStat(path, version)
file, err := CachePath(module.Version{Path: path, Version: version}, "info")
if err != nil {
return "", err
}
@ -349,6 +347,9 @@ func GoMod(path, rev string) ([]byte, error) {
if _, info, err := readDiskStat(path, rev); err == nil {
rev = info.Version
} else {
if errors.Is(err, statCacheErr) {
return nil, err
}
err := TryProxies(func(proxy string) error {
info, err := Lookup(proxy, path).Stat(rev)
if err == nil {
@ -384,7 +385,7 @@ func GoModFile(path, version string) (string, error) {
return "", err
}
// GoMod should have populated the disk cache for us.
file, _, err := readDiskGoMod(path, version)
file, err := CachePath(module.Version{Path: path, Version: version}, "mod")
if err != nil {
return "", err
}
@ -499,7 +500,7 @@ func readDiskStatByHash(path, rev string) (file string, info *RevInfo, err error
for _, name := range names {
if strings.HasSuffix(name, suffix) {
v := strings.TrimSuffix(name, ".info")
if IsPseudoVersion(v) && semver.Compare(v, maxVersion) > 0 {
if module.IsPseudoVersion(v) && semver.Compare(v, maxVersion) > 0 {
maxVersion = v
file, info, err = readDiskStat(path, strings.TrimSuffix(name, ".info"))
}
@ -547,7 +548,7 @@ func readDiskCache(path, rev, suffix string) (file string, data []byte, err erro
if err != nil {
return "", nil, errNotCached
}
data, err = renameio.ReadFile(file)
data, err = robustio.ReadFile(file)
if err != nil {
return file, nil, errNotCached
}
@ -584,7 +585,29 @@ func writeDiskCache(file string, data []byte) error {
return err
}
if err := renameio.WriteFile(file, data, 0666); err != nil {
// Write the file to a temporary location, and then rename it to its final
// path to reduce the likelihood of a corrupt file existing at that final path.
f, err := tempFile(filepath.Dir(file), filepath.Base(file), 0666)
if err != nil {
return err
}
defer func() {
// Only call os.Remove on f.Name() if we failed to rename it: otherwise,
// some other process may have created a new file with the same name after
// the rename completed.
if err != nil {
f.Close()
os.Remove(f.Name())
}
}()
if _, err := f.Write(data); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
if err := robustio.Rename(f.Name(), file); err != nil {
return err
}
@ -594,29 +617,49 @@ func writeDiskCache(file string, data []byte) error {
return nil
}
// tempFile creates a new temporary file with given permission bits.
func tempFile(dir, prefix string, perm fs.FileMode) (f *os.File, err error) {
for i := 0; i < 10000; i++ {
name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+".tmp")
f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
if os.IsExist(err) {
continue
}
break
}
return
}
// rewriteVersionList rewrites the version list in dir
// after a new *.mod file has been written.
func rewriteVersionList(dir string) {
func rewriteVersionList(dir string) (err error) {
if filepath.Base(dir) != "@v" {
base.Fatalf("go: internal error: misuse of rewriteVersionList")
}
listFile := filepath.Join(dir, "list")
// We use a separate lockfile here instead of locking listFile itself because
// we want to use Rename to write the file atomically. The list may be read by
// a GOPROXY HTTP server, and if we crash midway through a rewrite (or if the
// HTTP server ignores our locking and serves the file midway through a
// rewrite) it's better to serve a stale list than a truncated one.
unlock, err := lockedfile.MutexAt(listFile + ".lock").Lock()
// Lock listfile when writing to it to try to avoid corruption to the file.
// Under rare circumstances, for instance, if the system loses power in the
// middle of a write it is possible for corrupt data to be written. This is
// not a problem for the go command itself, but may be an issue if the the
// cache is being served by a GOPROXY HTTP server. This will be corrected
// the next time a new version of the module is fetched and the file is rewritten.
// TODO(matloob): golang.org/issue/43313 covers adding a go mod verify
// command that removes module versions that fail checksums. It should also
// remove list files that are detected to be corrupt.
f, err := lockedfile.Edit(listFile)
if err != nil {
base.Fatalf("go: can't lock version list lockfile: %v", err)
return err
}
defer unlock()
defer func() {
if cerr := f.Close(); cerr != nil && err == nil {
err = cerr
}
}()
infos, err := os.ReadDir(dir)
if err != nil {
return
return err
}
var list []string
for _, info := range infos {
@ -634,19 +677,74 @@ func rewriteVersionList(dir string) {
}
}
}
SortVersions(list)
semver.Sort(list)
var buf bytes.Buffer
for _, v := range list {
buf.WriteString(v)
buf.WriteString("\n")
}
old, _ := renameio.ReadFile(listFile)
if bytes.Equal(buf.Bytes(), old) {
return
if fi, err := f.Stat(); err == nil && int(fi.Size()) == buf.Len() {
old := make([]byte, buf.Len()+1)
if n, err := f.ReadAt(old, 0); err == io.EOF && n == buf.Len() && bytes.Equal(buf.Bytes(), old) {
return nil // No edit needed.
}
}
// Remove existing contents, so that when we truncate to the actual size it will zero-fill,
// and we will be able to detect (some) incomplete writes as files containing trailing NUL bytes.
if err := f.Truncate(0); err != nil {
return err
}
// Reserve the final size and zero-fill.
if err := f.Truncate(int64(buf.Len())); err != nil {
return err
}
// Write the actual contents. If this fails partway through,
// the remainder of the file should remain as zeroes.
if _, err := f.Write(buf.Bytes()); err != nil {
f.Truncate(0)
return err
}
if err := renameio.WriteFile(listFile, buf.Bytes(), 0666); err != nil {
base.Fatalf("go: failed to write version list: %v", err)
}
return nil
}
var (
statCacheOnce sync.Once
statCacheErr error
)
// checkCacheDir checks if the directory specified by GOMODCACHE exists. An
// error is returned if it does not.
func checkCacheDir() error {
if cfg.GOMODCACHE == "" {
// modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE
// is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen.
return fmt.Errorf("internal error: cfg.GOMODCACHE not set")
}
if !filepath.IsAbs(cfg.GOMODCACHE) {
return fmt.Errorf("GOMODCACHE entry is relative; must be absolute path: %q.\n", cfg.GOMODCACHE)
}
// os.Stat is slow on Windows, so we only call it once to prevent unnecessary
// I/O every time this function is called.
statCacheOnce.Do(func() {
fi, err := os.Stat(cfg.GOMODCACHE)
if err != nil {
if !os.IsNotExist(err) {
statCacheErr = fmt.Errorf("could not create module cache: %w", err)
return
}
if err := os.MkdirAll(cfg.GOMODCACHE, 0777); err != nil {
statCacheErr = fmt.Errorf("could not create module cache: %w", err)
return
}
return
}
if !fi.IsDir() {
statCacheErr = fmt.Errorf("could not create module cache: %q is not a directory", cfg.GOMODCACHE)
return
}
})
return statCacheErr
}

6
libgo/go/cmd/go/internal/modfetch/codehost/git.go
View file

@ -296,6 +296,9 @@ func (r *gitRepo) stat(rev string) (*RevInfo, error) {
// Or maybe it's the prefix of a hash of a named ref.
// Try to resolve to both a ref (git name) and full (40-hex-digit) commit hash.
r.refsOnce.Do(r.loadRefs)
// loadRefs may return an error if git fails, for example segfaults, or
// could not load a private repo, but defer checking to the else block
// below, in case we already have the rev in question in the local cache.
var ref, hash string
if r.refs["refs/tags/"+rev] != "" {
ref = "refs/tags/" + rev
@ -332,6 +335,9 @@ func (r *gitRepo) stat(rev string) (*RevInfo, error) {
hash = rev
}
} else {
if r.refsErr != nil {
return nil, r.refsErr
}
return nil, &UnknownRevisionError{Rev: rev}
}

49
libgo/go/cmd/go/internal/modfetch/codehost/git_test.go
View file

@ -8,7 +8,6 @@ import (
"archive/zip"
"bytes"
"flag"
"fmt"
"internal/testenv"
"io"
"io/fs"
@ -47,12 +46,6 @@ var altRepos = []string{
var localGitRepo string
func testMain(m *testing.M) int {
if _, err := exec.LookPath("git"); err != nil {
fmt.Fprintln(os.Stderr, "skipping because git binary not found")
fmt.Println("PASS")
return 0
}
dir, err := os.MkdirTemp("", "gitrepo-test-")
if err != nil {
log.Fatal(err)
@ -60,23 +53,25 @@ func testMain(m *testing.M) int {
defer os.RemoveAll(dir)
if testenv.HasExternalNetwork() && testenv.HasExec() {
// Clone gitrepo1 into a local directory.
// If we use a file:// URL to access the local directory,
// then git starts up all the usual protocol machinery,
// which will let us test remote git archive invocations.
localGitRepo = filepath.Join(dir, "gitrepo2")
if _, err := Run("", "git", "clone", "--mirror", gitrepo1, localGitRepo); err != nil {
log.Fatal(err)
}
if _, err := Run(localGitRepo, "git", "config", "daemon.uploadarch", "true"); err != nil {
log.Fatal(err)
if _, err := exec.LookPath("git"); err == nil {
// Clone gitrepo1 into a local directory.
// If we use a file:// URL to access the local directory,
// then git starts up all the usual protocol machinery,
// which will let us test remote git archive invocations.
localGitRepo = filepath.Join(dir, "gitrepo2")
if _, err := Run("", "git", "clone", "--mirror", gitrepo1, localGitRepo); err != nil {
log.Fatal(err)
}
if _, err := Run(localGitRepo, "git", "config", "daemon.uploadarch", "true"); err != nil {
log.Fatal(err)
}
}
}
return m.Run()
}
func testRepo(remote string) (Repo, error) {
func testRepo(t *testing.T, remote string) (Repo, error) {
if remote == "localGitRepo" {
// Convert absolute path to file URL. LocalGitRepo will not accept
// Windows absolute paths because they look like a host:path remote.
@ -87,15 +82,17 @@ func testRepo(remote string) (Repo, error) {
} else {
url = "file:///" + filepath.ToSlash(localGitRepo)
}
testenv.MustHaveExecPath(t, "git")
return LocalGitRepo(url)
}
kind := "git"
vcs := "git"
for _, k := range []string{"hg"} {
if strings.Contains(remote, "/"+k+"/") {
kind = k
vcs = k
}
}
return NewRepo(kind, remote)
testenv.MustHaveExecPath(t, vcs)
return NewRepo(vcs, remote)
}
var tagsTests = []struct {
@ -116,7 +113,7 @@ func TestTags(t *testing.T) {
for _, tt := range tagsTests {
f := func(t *testing.T) {
r, err := testRepo(tt.repo)
r, err := testRepo(t, tt.repo)
if err != nil {
t.Fatal(err)
}
@ -168,7 +165,7 @@ func TestLatest(t *testing.T) {
for _, tt := range latestTests {
f := func(t *testing.T) {
r, err := testRepo(tt.repo)
r, err := testRepo(t, tt.repo)
if err != nil {
t.Fatal(err)
}
@ -221,7 +218,7 @@ func TestReadFile(t *testing.T) {
for _, tt := range readFileTests {
f := func(t *testing.T) {
r, err := testRepo(tt.repo)
r, err := testRepo(t, tt.repo)
if err != nil {
t.Fatal(err)
}
@ -412,7 +409,7 @@ func TestReadZip(t *testing.T) {
for _, tt := range readZipTests {
f := func(t *testing.T) {
r, err := testRepo(tt.repo)
r, err := testRepo(t, tt.repo)
if err != nil {
t.Fatal(err)
}
@ -581,7 +578,7 @@ func TestStat(t *testing.T) {
for _, tt := range statTests {
f := func(t *testing.T) {
r, err := testRepo(tt.repo)
r, err := testRepo(t, tt.repo)
if err != nil {
t.Fatal(err)
}

1
libgo/go/cmd/go/internal/modfetch/codehost/shell.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build ignore
// +build ignore
// Interactive debugging shell for codehost.Repo implementations.

53
libgo/go/cmd/go/internal/modfetch/coderepo.go
View file

@ -159,7 +159,7 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) {
if r.codeDir != "" {
v = v[len(r.codeDir)+1:]
}
if v == "" || v != module.CanonicalVersion(v) || IsPseudoVersion(v) {
if v == "" || v != module.CanonicalVersion(v) || module.IsPseudoVersion(v) {
continue
}
@ -172,8 +172,8 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) {
list = append(list, v)
}
SortVersions(list)
SortVersions(incompatible)
semver.Sort(list)
semver.Sort(incompatible)
return r.appendIncompatibleVersions(list, incompatible)
}
@ -385,7 +385,7 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
if statVers != "" && statVers == module.CanonicalVersion(statVers) {
info2.Version = statVers
if IsPseudoVersion(info2.Version) {
if module.IsPseudoVersion(info2.Version) {
if err := r.validatePseudoVersion(info, info2.Version); err != nil {
return nil, err
}
@ -433,7 +433,7 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
}
trimmed := tag[len(tagPrefix):]
// Tags that look like pseudo-versions would be confusing. Ignore them.
if IsPseudoVersion(tag) {
if module.IsPseudoVersion(tag) {
return "", false
}
@ -531,7 +531,7 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
pseudoBase, _ = tagToVersion(tag) // empty if the tag is invalid
}
info2.Version = PseudoVersion(r.pseudoMajor, pseudoBase, info.Time, info.Short)
info2.Version = module.PseudoVersion(r.pseudoMajor, pseudoBase, info.Time, info.Short)
return checkGoMod()
}
@ -560,7 +560,7 @@ func (r *codeRepo) validatePseudoVersion(info *codehost.RevInfo, version string)
return err
}
rev, err := PseudoVersionRev(version)
rev, err := module.PseudoVersionRev(version)
if err != nil {
return err
}
@ -575,12 +575,12 @@ func (r *codeRepo) validatePseudoVersion(info *codehost.RevInfo, version string)
}
}
t, err := PseudoVersionTime(version)
t, err := module.PseudoVersionTime(version)
if err != nil {
return err
}
if !t.Equal(info.Time.Truncate(time.Second)) {
return fmt.Errorf("does not match version-control timestamp (expected %s)", info.Time.UTC().Format(pseudoVersionTimestampFormat))
return fmt.Errorf("does not match version-control timestamp (expected %s)", info.Time.UTC().Format(module.PseudoVersionTimestampFormat))
}
tagPrefix := ""
@ -604,7 +604,7 @@ func (r *codeRepo) validatePseudoVersion(info *codehost.RevInfo, version string)
// not enforce that property when resolving existing pseudo-versions: we don't
// know when the parent tags were added, and the highest-tagged parent may not
// have existed when the pseudo-version was first resolved.
base, err := PseudoVersionBase(strings.TrimSuffix(version, "+incompatible"))
base, err := module.PseudoVersionBase(strings.TrimSuffix(version, "+incompatible"))
if err != nil {
return err
}
@ -661,7 +661,7 @@ func (r *codeRepo) validatePseudoVersion(info *codehost.RevInfo, version string)
if err != nil {
return err
}
rev, err := PseudoVersionRev(version)
rev, err := module.PseudoVersionRev(version)
if err != nil {
return fmt.Errorf("not a descendent of preceding tag (%s)", lastTag)
}
@ -672,8 +672,8 @@ func (r *codeRepo) validatePseudoVersion(info *codehost.RevInfo, version string)
func (r *codeRepo) revToRev(rev string) string {
if semver.IsValid(rev) {
if IsPseudoVersion(rev) {
r, _ := PseudoVersionRev(rev)
if module.IsPseudoVersion(rev) {
r, _ := module.PseudoVersionRev(rev)
return r
}
if semver.Build(rev) == "+incompatible" {
@ -843,7 +843,7 @@ func (r *codeRepo) GoMod(version string) (data []byte, err error) {
return nil, fmt.Errorf("version %s is not canonical", version)
}
if IsPseudoVersion(version) {
if module.IsPseudoVersion(version) {
// findDir ignores the metadata encoded in a pseudo-version,
// only using the revision at the end.
// Invoke Stat to verify the metadata explicitly so we don't return
@ -864,22 +864,25 @@ func (r *codeRepo) GoMod(version string) (data []byte, err error) {
data, err = r.code.ReadFile(rev, path.Join(dir, "go.mod"), codehost.MaxGoMod)
if err != nil {
if os.IsNotExist(err) {
return r.legacyGoMod(rev, dir), nil
return LegacyGoMod(r.modPath), nil
}
return nil, err
}
return data, nil
}
func (r *codeRepo) legacyGoMod(rev, dir string) []byte {
// We used to try to build a go.mod reflecting pre-existing
// package management metadata files, but the conversion
// was inherently imperfect (because those files don't have
// exactly the same semantics as go.mod) and, when done
// for dependencies in the middle of a build, impossible to
// correct. So we stopped.
// Return a fake go.mod that simply declares the module path.
return []byte(fmt.Sprintf("module %s\n", modfile.AutoQuote(r.modPath)))
// LegacyGoMod generates a fake go.mod file for a module that doesn't have one.
// The go.mod file contains a module directive and nothing else: no go version,
// no requirements.
//
// We used to try to build a go.mod reflecting pre-existing
// package management metadata files, but the conversion
// was inherently imperfect (because those files don't have
// exactly the same semantics as go.mod) and, when done
// for dependencies in the middle of a build, impossible to
// correct. So we stopped.
func LegacyGoMod(modPath string) []byte {
return []byte(fmt.Sprintf("module %s\n", modfile.AutoQuote(modPath)))
}
func (r *codeRepo) modPrefix(rev string) string {
@ -942,7 +945,7 @@ func (r *codeRepo) Zip(dst io.Writer, version string) error {
return fmt.Errorf("version %s is not canonical", version)
}
if IsPseudoVersion(version) {
if module.IsPseudoVersion(version) {
// findDir ignores the metadata encoded in a pseudo-version,
// only using the revision at the end.
// Invoke Stat to verify the metadata explicitly so we don't return

101
libgo/go/cmd/go/internal/modfetch/fetch.go
View file

@ -8,6 +8,8 @@ import (
"archive/zip"
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"errors"
"fmt"
"io"
@ -20,9 +22,9 @@ import (
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/fsys"
"cmd/go/internal/lockedfile"
"cmd/go/internal/par"
"cmd/go/internal/renameio"
"cmd/go/internal/robustio"
"cmd/go/internal/trace"
@ -37,10 +39,8 @@ var downloadCache par.Cache
// local download cache and returns the name of the directory
// corresponding to the root of the module's file tree.
func Download(ctx context.Context, mod module.Version) (dir string, err error) {
if cfg.GOMODCACHE == "" {
// modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE
// is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen.
base.Fatalf("go: internal error: cfg.GOMODCACHE not set")
if err := checkCacheDir(); err != nil {
base.Fatalf("go: %v", err)
}
// The par.Cache here avoids duplicate work.
@ -223,11 +223,10 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e
// Clean up any remaining tempfiles from previous runs.
// This is only safe to do because the lock file ensures that their
// writers are no longer active.
for _, base := range []string{zipfile, zipfile + "hash"} {
if old, err := filepath.Glob(renameio.Pattern(base)); err == nil {
for _, path := range old {
os.Remove(path) // best effort
}
tmpPattern := filepath.Base(zipfile) + "*.tmp"
if old, err := filepath.Glob(filepath.Join(filepath.Dir(zipfile), tmpPattern)); err == nil {
for _, path := range old {
os.Remove(path) // best effort
}
}
@ -242,7 +241,7 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e
// contents of the file (by hashing it) before we commit it. Because the file
// is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
// validate it: we can't just tee the stream as we write it.
f, err := os.CreateTemp(filepath.Dir(zipfile), filepath.Base(renameio.Pattern(zipfile)))
f, err := os.CreateTemp(filepath.Dir(zipfile), tmpPattern)
if err != nil {
return err
}
@ -298,12 +297,6 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e
}
}
// Sync the file before renaming it: otherwise, after a crash the reader may
// observe a 0-length file instead of the actual contents.
// See https://golang.org/issue/22397#issuecomment-380831736.
if err := f.Sync(); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
@ -334,7 +327,21 @@ func hashZip(mod module.Version, zipfile, ziphashfile string) error {
if err := checkModSum(mod, hash); err != nil {
return err
}
return renameio.WriteFile(ziphashfile, []byte(hash), 0666)
hf, err := lockedfile.Create(ziphashfile)
if err != nil {
return err
}
if err := hf.Truncate(int64(len(hash))); err != nil {
return err
}
if _, err := hf.WriteAt([]byte(hash), 0); err != nil {
return err
}
if err := hf.Close(); err != nil {
return err
}
return nil
}
// makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
@ -410,7 +417,18 @@ func initGoSum() (bool, error) {
goSum.m = make(map[module.Version][]string)
goSum.status = make(map[modSum]modSumStatus)
data, err := lockedfile.Read(GoSumFile)
var (
data []byte
err error
)
if actualSumFile, ok := fsys.OverlayPath(GoSumFile); ok {
// Don't lock go.sum if it's part of the overlay.
// On Plan 9, locking requires chmod, and we don't want to modify any file
// in the overlay. See #44700.
data, err = os.ReadFile(actualSumFile)
} else {
data, err = lockedfile.Read(GoSumFile)
}
if err != nil && !os.IsNotExist(err) {
return false, err
}
@ -485,11 +503,24 @@ func checkMod(mod module.Version) {
if err != nil {
base.Fatalf("verifying %v", module.VersionError(mod, err))
}
data, err := renameio.ReadFile(ziphash)
data, err := lockedfile.Read(ziphash)
if err != nil {
base.Fatalf("verifying %v", module.VersionError(mod, err))
}
h := strings.TrimSpace(string(data))
data = bytes.TrimSpace(data)
if !isValidSum(data) {
// Recreate ziphash file from zip file and use that to check the mod sum.
zip, err := CachePath(mod, "zip")
if err != nil {
base.Fatalf("verifying %v", module.VersionError(mod, err))
}
err = hashZip(mod, zip, ziphash)
if err != nil {
base.Fatalf("verifying %v", module.VersionError(mod, err))
}
return
}
h := string(data)
if !strings.HasPrefix(h, "h1:") {
base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
}
@ -634,11 +665,32 @@ func Sum(mod module.Version) string {
if err != nil {
return ""
}
data, err := renameio.ReadFile(ziphash)
data, err := lockedfile.Read(ziphash)
if err != nil {
return ""
}
return strings.TrimSpace(string(data))
data = bytes.TrimSpace(data)
if !isValidSum(data) {
return ""
}
return string(data)
}
// isValidSum returns true if data is the valid contents of a zip hash file.
// Certain critical files are written to disk by first truncating
// then writing the actual bytes, so that if the write fails
// the corrupt file should contain at least one of the null
// bytes written by the truncate operation.
func isValidSum(data []byte) bool {
if bytes.IndexByte(data, '\000') >= 0 {
return false
}
if len(data) != len("h1:")+base64.StdEncoding.EncodedLen(sha256.Size) {
return false
}
return true
}
// WriteGoSum writes the go.sum file if it needs to be updated.
@ -676,6 +728,9 @@ Outer:
if cfg.BuildMod == "readonly" {
base.Fatalf("go: updates to go.sum needed, disabled by -mod=readonly")
}
if _, ok := fsys.OverlayPath(GoSumFile); ok {
base.Fatalf("go: updates to go.sum needed, but go.sum is part of the overlay specified with -overlay")
}
// Make a best-effort attempt to acquire the side lock, only to exclude
// previous versions of the 'go' command from making simultaneous edits.

16
libgo/go/cmd/go/internal/modfetch/insecure.go
View file

@ -1,16 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package modfetch
import (
"cmd/go/internal/cfg"
"golang.org/x/mod/module"
)
// allowInsecure reports whether we are allowed to fetch this path in an insecure manner.
func allowInsecure(path string) bool {
return cfg.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, path)
}

10
libgo/go/cmd/go/internal/modfetch/proxy.go
View file

@ -228,7 +228,7 @@ func (p *proxyRepo) versionError(version string, err error) error {
Path: p.path,
Err: &module.InvalidVersionError{
Version: version,
Pseudo: IsPseudoVersion(version),
Pseudo: module.IsPseudoVersion(version),
Err: err,
},
}
@ -276,11 +276,11 @@ func (p *proxyRepo) Versions(prefix string) ([]string, error) {
var list []string
for _, line := range strings.Split(string(data), "\n") {
f := strings.Fields(line)
if len(f) >= 1 && semver.IsValid(f[0]) && strings.HasPrefix(f[0], prefix) && !IsPseudoVersion(f[0]) {
if len(f) >= 1 && semver.IsValid(f[0]) && strings.HasPrefix(f[0], prefix) && !module.IsPseudoVersion(f[0]) {
list = append(list, f[0])
}
}
SortVersions(list)
semver.Sort(list)
return list, nil
}
@ -307,8 +307,8 @@ func (p *proxyRepo) latest() (*RevInfo, error) {
)
if len(f) >= 2 {
ft, _ = time.Parse(time.RFC3339, f[1])
} else if IsPseudoVersion(f[0]) {
ft, _ = PseudoVersionTime(f[0])
} else if module.IsPseudoVersion(f[0]) {
ft, _ = module.PseudoVersionTime(f[0])
ftIsFromPseudo = true
} else {
// Repo.Latest promises that this method is only called where there are

154
libgo/go/cmd/go/internal/modfetch/pseudo_test.go
View file

@ -1,154 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package modfetch
import (
"testing"
"time"
)
var pseudoTests = []struct {
major string
older string
version string
}{
{"", "", "v0.0.0-20060102150405-hash"},
{"v0", "", "v0.0.0-20060102150405-hash"},
{"v1", "", "v1.0.0-20060102150405-hash"},
{"v2", "", "v2.0.0-20060102150405-hash"},
{"unused", "v0.0.0", "v0.0.1-0.20060102150405-hash"},
{"unused", "v1.2.3", "v1.2.4-0.20060102150405-hash"},
{"unused", "v1.2.99999999999999999", "v1.2.100000000000000000-0.20060102150405-hash"},
{"unused", "v1.2.3-pre", "v1.2.3-pre.0.20060102150405-hash"},
{"unused", "v1.3.0-pre", "v1.3.0-pre.0.20060102150405-hash"},
{"unused", "v0.0.0--", "v0.0.0--.0.20060102150405-hash"},
{"unused", "v1.0.0+metadata", "v1.0.1-0.20060102150405-hash+metadata"},
{"unused", "v2.0.0+incompatible", "v2.0.1-0.20060102150405-hash+incompatible"},
{"unused", "v2.3.0-pre+incompatible", "v2.3.0-pre.0.20060102150405-hash+incompatible"},
}
var pseudoTime = time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC)
func TestPseudoVersion(t *testing.T) {
for _, tt := range pseudoTests {
v := PseudoVersion(tt.major, tt.older, pseudoTime, "hash")
if v != tt.version {
t.Errorf("PseudoVersion(%q, %q, ...) = %v, want %v", tt.major, tt.older, v, tt.version)
}
}
}
func TestIsPseudoVersion(t *testing.T) {
for _, tt := range pseudoTests {
if !IsPseudoVersion(tt.version) {
t.Errorf("IsPseudoVersion(%q) = false, want true", tt.version)
}
if IsPseudoVersion(tt.older) {
t.Errorf("IsPseudoVersion(%q) = true, want false", tt.older)
}
}
}
func TestPseudoVersionTime(t *testing.T) {
for _, tt := range pseudoTests {
tm, err := PseudoVersionTime(tt.version)
if tm != pseudoTime || err != nil {
t.Errorf("PseudoVersionTime(%q) = %v, %v, want %v, nil", tt.version, tm.Format(time.RFC3339), err, pseudoTime.Format(time.RFC3339))
}
tm, err = PseudoVersionTime(tt.older)
if tm != (time.Time{}) || err == nil {
t.Errorf("PseudoVersionTime(%q) = %v, %v, want %v, error", tt.older, tm.Format(time.RFC3339), err, time.Time{}.Format(time.RFC3339))
}
}
}
func TestInvalidPseudoVersionTime(t *testing.T) {
const v = "---"
if _, err := PseudoVersionTime(v); err == nil {
t.Error("expected error, got nil instead")
}
}
func TestPseudoVersionRev(t *testing.T) {
for _, tt := range pseudoTests {
rev, err := PseudoVersionRev(tt.version)
if rev != "hash" || err != nil {
t.Errorf("PseudoVersionRev(%q) = %q, %v, want %q, nil", tt.older, rev, err, "hash")
}
rev, err = PseudoVersionRev(tt.older)
if rev != "" || err == nil {
t.Errorf("PseudoVersionRev(%q) = %q, %v, want %q, error", tt.older, rev, err, "")
}
}
}
func TestPseudoVersionBase(t *testing.T) {
for _, tt := range pseudoTests {
base, err := PseudoVersionBase(tt.version)
if err != nil {
t.Errorf("PseudoVersionBase(%q): %v", tt.version, err)
} else if base != tt.older {
t.Errorf("PseudoVersionBase(%q) = %q; want %q", tt.version, base, tt.older)
}
}
}
func TestInvalidPseudoVersionBase(t *testing.T) {
for _, in := range []string{
"v0.0.0",
"v0.0.0-", // malformed: empty prerelease
"v0.0.0-0.20060102150405-hash", // Z+1 == 0
"v0.1.0-0.20060102150405-hash", // Z+1 == 0
"v1.0.0-0.20060102150405-hash", // Z+1 == 0
"v0.0.0-20060102150405-hash+incompatible", // "+incompatible without base version
"v0.0.0-20060102150405-hash+metadata", // other metadata without base version
} {
base, err := PseudoVersionBase(in)
if err == nil || base != "" {
t.Errorf(`PseudoVersionBase(%q) = %q, %v; want "", error`, in, base, err)
}
}
}
func TestIncDecimal(t *testing.T) {
cases := []struct {
in, want string
}{
{"0", "1"},
{"1", "2"},
{"99", "100"},
{"100", "101"},
{"101", "102"},
}
for _, tc := range cases {
got := incDecimal(tc.in)
if got != tc.want {
t.Fatalf("incDecimal(%q) = %q; want %q", tc.in, tc.want, got)
}
}
}
func TestDecDecimal(t *testing.T) {
cases := []struct {
in, want string
}{
{"", ""},
{"0", ""},
{"00", ""},
{"1", "0"},
{"2", "1"},
{"99", "98"},
{"100", "99"},
{"101", "100"},
}
for _, tc := range cases {
got := decDecimal(tc.in)
if got != tc.want {
t.Fatalf("decDecimal(%q) = %q; want %q", tc.in, tc.want, got)
}
}
}

75
libgo/go/cmd/go/internal/modfetch/repo.go
View file

@ -9,7 +9,6 @@ import (
"io"
"io/fs"
"os"
"sort"
"strconv"
"time"
@ -20,7 +19,6 @@ import (
web "cmd/go/internal/web"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
)
const traceRepo = false // trace all repo actions, for debugging
@ -35,7 +33,7 @@ type Repo interface {
// Pseudo-versions are not included.
//
// Versions should be returned sorted in semver order
// (implementations can use SortVersions).
// (implementations can use semver.Sort).
//
// Versions returns a non-nil error only if there was a problem
// fetching the list of versions: it may return an empty list
@ -171,15 +169,6 @@ type RevInfo struct {
// and it can check that the path can be resolved to a target repository.
// To avoid version control access except when absolutely necessary,
// Lookup does not attempt to connect to the repository itself.
//
// The ImportRepoRev function is a variant of Import which is limited
// to code in a source code repository at a particular revision identifier
// (usually a commit hash or source code repository tag, not necessarily
// a module version).
// ImportRepoRev is used when converting legacy dependency requirements
// from older systems into go.mod files. Those older systems worked
// at either package or repository granularity, and most of the time they
// recorded commit hashes, not tagged versions.
var lookupCache par.Cache
@ -194,7 +183,8 @@ type lookupCacheKey struct {
// from its origin, and "noproxy" indicates that the patch should be fetched
// directly only if GONOPROXY matches the given path.
//
// For the distinguished proxy "off", Lookup always returns a non-nil error.
// For the distinguished proxy "off", Lookup always returns a Repo that returns
// a non-nil error for every method call.
//
// A successful return does not guarantee that the module
// has any defined versions.
@ -267,7 +257,7 @@ var (
func lookupDirect(path string) (Repo, error) {
security := web.SecureOnly
if allowInsecure(path) {
if module.MatchPrefixPatterns(cfg.GOINSECURE, path) {
security = web.Insecure
}
rr, err := vcs.RepoRootForImportPath(path, vcs.PreferMod, security)
@ -299,63 +289,6 @@ func lookupCodeRepo(rr *vcs.RepoRoot) (codehost.Repo, error) {
return code, nil
}
// ImportRepoRev returns the module and version to use to access
// the given import path loaded from the source code repository that
// the original "go get" would have used, at the specific repository revision
// (typically a commit hash, but possibly also a source control tag).
func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) {
if cfg.BuildMod == "vendor" || cfg.BuildMod == "readonly" {
return nil, nil, fmt.Errorf("repo version lookup disabled by -mod=%s", cfg.BuildMod)
}
// Note: Because we are converting a code reference from a legacy
// version control system, we ignore meta tags about modules
// and use only direct source control entries (get.IgnoreMod).
security := web.SecureOnly
if allowInsecure(path) {
security = web.Insecure
}
rr, err := vcs.RepoRootForImportPath(path, vcs.IgnoreMod, security)
if err != nil {
return nil, nil, err
}
code, err := lookupCodeRepo(rr)
if err != nil {
return nil, nil, err
}
revInfo, err := code.Stat(rev)
if err != nil {
return nil, nil, err
}
// TODO: Look in repo to find path, check for go.mod files.
// For now we're just assuming rr.Root is the module path,
// which is true in the absence of go.mod files.
repo, err := newCodeRepo(code, rr.Root, rr.Root)
if err != nil {
return nil, nil, err
}
info, err := repo.(*codeRepo).convert(revInfo, rev)
if err != nil {
return nil, nil, err
}
return repo, info, nil
}
func SortVersions(list []string) {
sort.Slice(list, func(i, j int) bool {
cmp := semver.Compare(list[i], list[j])
if cmp != 0 {
return cmp < 0
}
return list[i] < list[j]
})
}
// A loggingRepo is a wrapper around an underlying Repo
// that prints a log message at the start and end of each call.
// It can be inserted when debugging.

5
libgo/go/cmd/go/internal/modfetch/sumdb.go
View file

@ -4,6 +4,7 @@
// Go checksum database lookup
//go:build !cmd_go_bootstrap
// +build !cmd_go_bootstrap
package modfetch
@ -33,7 +34,7 @@ import (
// useSumDB reports whether to use the Go checksum database for the given module.
func useSumDB(mod module.Version) bool {
return cfg.GOSUMDB != "off" && !cfg.Insecure && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path)
return cfg.GOSUMDB != "off" && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path)
}
// lookupSumDB returns the Go checksum database's go.sum lines for the given module,
@ -184,7 +185,7 @@ func (c *dbClient) initBase() {
}
})
if errors.Is(err, fs.ErrNotExist) {
// No proxies, or all proxies failed (with 404, 410, or were were allowed
// No proxies, or all proxies failed (with 404, 410, or were allowed
// to fall back), or we reached an explicit "direct" or "off".
c.base = c.direct
} else if err != nil {

396
libgo/go/cmd/go/internal/modget/get.go
View file

@ -30,16 +30,15 @@ import (
"fmt"
"os"
"path/filepath"
"reflect"
"runtime"
"sort"
"strings"
"sync"
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/imports"
"cmd/go/internal/load"
"cmd/go/internal/modfetch"
"cmd/go/internal/modload"
"cmd/go/internal/par"
"cmd/go/internal/search"
@ -53,7 +52,7 @@ import (
var CmdGet = &base.Command{
// Note: -d -u are listed explicitly because they are the most common get flags.
// Do not send CLs removing them because they're covered by [get flags].
UsageLine: "go get [-d] [-t] [-u] [-v] [-insecure] [build flags] [packages]",
UsageLine: "go get [-d] [-t] [-u] [-v] [build flags] [packages]",
Short: "add dependencies to current module and install them",
Long: `
Get resolves its command-line arguments to packages at specific module versions,
@ -99,14 +98,6 @@ but changes the default to select patch releases.
When the -t and -u flags are used together, get will update
test dependencies as well.
The -insecure flag permits fetching from repositories and resolving
custom domains using insecure schemes such as HTTP, and also bypassess
module sum validation using the checksum database. Use with caution.
This flag is deprecated and will be removed in a future version of go.
To permit the use of insecure schemes, use the GOINSECURE environment
variable instead. To bypass module sum validation, use GOPRIVATE or
GONOSUMDB. See 'go help environment' for details.
The -d flag instructs get not to build or install packages. get will only
update go.mod and download source code needed to build packages.
@ -227,13 +218,13 @@ variable for future go command invocations.
}
var (
getD = CmdGet.Flag.Bool("d", false, "")
getF = CmdGet.Flag.Bool("f", false, "")
getFix = CmdGet.Flag.Bool("fix", false, "")
getM = CmdGet.Flag.Bool("m", false, "")
getT = CmdGet.Flag.Bool("t", false, "")
getU upgradeFlag
// -insecure is cfg.Insecure
getD = CmdGet.Flag.Bool("d", false, "")
getF = CmdGet.Flag.Bool("f", false, "")
getFix = CmdGet.Flag.Bool("fix", false, "")
getM = CmdGet.Flag.Bool("m", false, "")
getT = CmdGet.Flag.Bool("t", false, "")
getU upgradeFlag
getInsecure = CmdGet.Flag.Bool("insecure", false, "")
// -v is cfg.BuildV
)
@ -264,7 +255,6 @@ func (v *upgradeFlag) String() string { return "" }
func init() {
work.AddBuildFlags(CmdGet, work.OmitModFlag)
CmdGet.Run = runGet // break init loop
CmdGet.Flag.BoolVar(&cfg.Insecure, "insecure", cfg.Insecure, "")
CmdGet.Flag.Var(&getU, "u", "")
}
@ -284,10 +274,9 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
if *getM {
base.Fatalf("go get: -m flag is no longer supported; consider -d to skip building packages")
}
if cfg.Insecure {
fmt.Fprintf(os.Stderr, "go get: -insecure flag is deprecated; see 'go help get' for details\n")
if *getInsecure {
base.Fatalf("go get: -insecure flag is no longer supported; use GOINSECURE instead")
}
load.ModResolveTests = *getT
// Do not allow any updating of go.mod until we've applied
// all the requested changes and checked that the result matches
@ -298,8 +287,6 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
// 'go get' is expected to do this, unlike other commands.
modload.AllowMissingModuleImports()
modload.LoadModFile(ctx) // Initializes modload.Target.
queries := parseArgs(ctx, args)
r := newResolver(ctx, queries)
@ -310,7 +297,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
r.performWildcardQueries(ctx)
r.performPatternAllQueries(ctx)
if changed := r.resolveCandidates(ctx, queries, nil); changed {
if changed := r.resolveQueries(ctx, queries); changed {
// 'go get' arguments can be (and often are) package patterns rather than
// (just) modules. A package can be provided by any module with a prefix
// of its import path, and a wildcard can even match packages in modules
@ -347,12 +334,12 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
// - ambiguous import errors.
// TODO(#27899): Try to resolve ambiguous import errors automatically.
upgrades := r.findAndUpgradeImports(ctx, queries)
if changed := r.resolveCandidates(ctx, nil, upgrades); changed {
if changed := r.applyUpgrades(ctx, upgrades); changed {
continue
}
r.findMissingWildcards(ctx)
if changed := r.resolveCandidates(ctx, r.wildcardQueries, nil); changed {
if changed := r.resolveQueries(ctx, r.wildcardQueries); changed {
continue
}
@ -367,7 +354,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
pkgPatterns = append(pkgPatterns, q.pattern)
}
}
r.checkPackagesAndRetractions(ctx, pkgPatterns)
r.checkPackageProblems(ctx, pkgPatterns)
// We've already downloaded modules (and identified direct and indirect
// dependencies) by loading packages in findAndUpgradeImports.
@ -380,12 +367,51 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
// directory.
if !*getD && len(pkgPatterns) > 0 {
work.BuildInit()
pkgs := load.PackagesAndErrors(ctx, pkgPatterns)
pkgOpts := load.PackageOpts{ModResolveTests: *getT}
var pkgs []*load.Package
for _, pkg := range load.PackagesAndErrors(ctx, pkgOpts, pkgPatterns) {
if pkg.Error != nil {
var noGo *load.NoGoError
if errors.As(pkg.Error.Err, &noGo) {
if m := modload.PackageModule(pkg.ImportPath); m.Path == pkg.ImportPath {
// pkg is at the root of a module, and doesn't exist with the current
// build tags. Probably the user just wanted to change the version of
// that module — not also build the package — so suppress the error.
// (See https://golang.org/issue/33526.)
continue
}
}
}
pkgs = append(pkgs, pkg)
}
load.CheckPackageErrors(pkgs)
haveExternalExe := false
for _, pkg := range pkgs {
if pkg.Name == "main" && pkg.Module != nil && pkg.Module.Path != modload.Target.Path {
haveExternalExe = true
break
}
}
if haveExternalExe {
fmt.Fprint(os.Stderr, "go get: installing executables with 'go get' in module mode is deprecated.")
var altMsg string
if modload.HasModRoot() {
altMsg = `
To adjust and download dependencies of the current module, use 'go get -d'.
To install using requirements of the current module, use 'go install'.
To install ignoring the current module, use 'go install' with a version,
like 'go install example.com/cmd@latest'.
`
} else {
altMsg = "\n\tUse 'go install pkg@version' instead.\n"
}
fmt.Fprint(os.Stderr, altMsg)
fmt.Fprintf(os.Stderr, "\tFor more information, see https://golang.org/doc/go-get-install-deprecation\n\tor run 'go help get' or 'go help install'.\n")
}
work.InstallPackages(ctx, pkgPatterns, pkgs)
// TODO(#40276): After Go 1.16, print a deprecation notice when building and
// installing main packages. 'go install pkg' or 'go install pkg@version'
// should be used instead. Give the specific argument to use if possible.
}
if !modload.HasModRoot() {
@ -396,7 +422,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
oldReqs := reqsFromGoMod(modload.ModFile())
modload.AllowWriteGoMod()
modload.WriteGoMod()
modload.WriteGoMod(ctx)
modload.DisallowWriteGoMod()
newReqs := reqsFromGoMod(modload.ModFile())
@ -460,9 +486,8 @@ type resolver struct {
// that resolved the module to that version (the “reason”).
resolvedVersion map[string]versionReason
buildList []module.Version
buildListResolvedVersions int // len(resolvedVersion) when buildList was computed
buildListVersion map[string]string // index of buildList (module path → version)
buildList []module.Version
buildListVersion map[string]string // index of buildList (module path → version)
initialVersion map[string]string // index of the initial build list at the start of 'go get'
@ -479,7 +504,12 @@ type versionReason struct {
}
func newResolver(ctx context.Context, queries []*query) *resolver {
buildList := modload.LoadAllModules(ctx)
// LoadModGraph also sets modload.Target, which is needed by various resolver
// methods.
const defaultGoVersion = ""
mg := modload.LoadModGraph(ctx, defaultGoVersion)
buildList := mg.BuildList()
initialVersion := make(map[string]string, len(buildList))
for _, m := range buildList {
initialVersion[m.Path] = m.Version
@ -688,7 +718,7 @@ func (r *resolver) performLocalQueries(ctx context.Context) {
// Absolute paths like C:\foo and relative paths like ../foo... are
// restricted to matching packages in the main module.
pkgPattern := modload.DirImportPath(q.pattern)
pkgPattern := modload.DirImportPath(ctx, q.pattern)
if pkgPattern == "." {
return errSet(fmt.Errorf("%s%s is not within module rooted at %s", q.pattern, absDetail, modload.ModRoot()))
}
@ -1121,9 +1151,11 @@ func (r *resolver) findAndUpgradeImports(ctx context.Context, queries []*query)
// build list.
func (r *resolver) loadPackages(ctx context.Context, patterns []string, findPackage func(ctx context.Context, path string, m module.Version) (versionOk bool)) {
opts := modload.PackageOpts{
Tags: imports.AnyTags(),
LoadTests: *getT,
SilenceErrors: true, // May be fixed by subsequent upgrades or downgrades.
Tags: imports.AnyTags(),
VendorModulesInGOROOTSrc: true,
LoadTests: *getT,
AssumeRootsImported: true, // After 'go get foo', imports of foo should build.
SilencePackageErrors: true, // May be fixed by subsequent upgrades or downgrades.
}
opts.AllowPackage = func(ctx context.Context, path string, m module.Version) error {
@ -1176,24 +1208,19 @@ func (r *resolver) loadPackages(ctx context.Context, patterns []string, findPack
// to be updated before its dependencies can be loaded.
var errVersionChange = errors.New("version change needed")
// resolveCandidates resolves candidates sets that are attached to the given
// resolveQueries resolves candidate sets that are attached to the given
// queries and/or needed to provide the given missing-package dependencies.
//
// resolveCandidates starts by resolving one module version from each
// resolveQueries starts by resolving one module version from each
// unambiguous pathSet attached to the given queries.
//
// If no unambiguous query results in a change to the build list,
// resolveCandidates modifies the build list by adding one module version from
// each pathSet in missing, but does not mark those versions as resolved
// (so they can still be modified by other queries).
//
// If that still does not result in any changes to the build list,
// resolveCandidates revisits the ambiguous query candidates and resolves them
// resolveQueries revisits the ambiguous query candidates and resolves them
// arbitrarily in order to guarantee forward progress.
//
// If all pathSets are resolved without any changes to the build list,
// resolveCandidates returns with changed=false.
func (r *resolver) resolveCandidates(ctx context.Context, queries []*query, upgrades []pathSet) (changed bool) {
// resolveQueries returns with changed=false.
func (r *resolver) resolveQueries(ctx context.Context, queries []*query) (changed bool) {
defer base.ExitIfErrors()
// Note: this is O(N²) with the number of pathSets in the worst case.
@ -1247,13 +1274,53 @@ func (r *resolver) resolveCandidates(ctx context.Context, queries []*query, upgr
}
}
if changed := r.updateBuildList(ctx, nil); changed {
// The build list has changed, so disregard any missing packages: they might
// now be determined by requirements in the build list, which we would
// prefer to use instead of arbitrary "latest" versions.
return true
if resolved > 0 {
if changed = r.updateBuildList(ctx, nil); changed {
// The build list has changed, so disregard any remaining ambiguous queries:
// they might now be determined by requirements in the build list, which we
// would prefer to use instead of arbitrary versions.
return true
}
}
// The build list will be the same on the next iteration as it was on this
// iteration, so any ambiguous queries will remain so. In order to make
// progress, resolve them arbitrarily but deterministically.
//
// If that results in conflicting versions, the user can re-run 'go get'
// with additional explicit versions for the conflicting packages or
// modules.
resolvedArbitrarily := 0
for _, q := range queries {
for _, cs := range q.candidates {
isPackage, m := r.chooseArbitrarily(cs)
if isPackage {
q.matchesPackages = true
}
r.resolve(q, m)
resolvedArbitrarily++
}
}
if resolvedArbitrarily > 0 {
changed = r.updateBuildList(ctx, nil)
}
return changed
}
// applyUpgrades disambiguates candidate sets that are needed to upgrade (or
// provide) transitive dependencies imported by previously-resolved packages.
//
// applyUpgrades modifies the build list by adding one module version from each
// pathSet in upgrades, then downgrading (or further upgrading) those modules as
// needed to maintain any already-resolved versions of other modules.
// applyUpgrades does not mark the new versions as resolved, so they can still
// be further modified by other queries (such as wildcards).
//
// If all pathSets are resolved without any changes to the build list,
// applyUpgrades returns with changed=false.
func (r *resolver) applyUpgrades(ctx context.Context, upgrades []pathSet) (changed bool) {
defer base.ExitIfErrors()
// Arbitrarily add a "latest" version that provides each missing package, but
// do not mark the version as resolved: we still want to allow the explicit
// queries to modify the resulting versions.
@ -1276,27 +1343,9 @@ func (r *resolver) resolveCandidates(ctx context.Context, queries []*query, upgr
tentative = append(tentative, m)
}
base.ExitIfErrors()
if changed := r.updateBuildList(ctx, tentative); changed {
return true
}
// The build list will be the same on the next iteration as it was on this
// iteration, so any ambiguous queries will remain so. In order to make
// progress, resolve them arbitrarily but deterministically.
//
// If that results in conflicting versions, the user can re-run 'go get'
// with additional explicit versions for the conflicting packages or
// modules.
for _, q := range queries {
for _, cs := range q.candidates {
isPackage, m := r.chooseArbitrarily(cs)
if isPackage {
q.matchesPackages = true
}
r.resolve(q, m)
}
}
return r.updateBuildList(ctx, nil)
changed = r.updateBuildList(ctx, tentative)
return changed
}
// disambiguate eliminates candidates from cs that conflict with other module
@ -1417,25 +1466,33 @@ func (r *resolver) chooseArbitrarily(cs pathSet) (isPackage bool, m module.Versi
return false, cs.mod
}
// checkPackagesAndRetractions reloads packages for the given patterns and
// reports missing and ambiguous package errors. It also reports loads and
// reports retractions for resolved modules and modules needed to build
// named packages.
// checkPackageProblems reloads packages for the given patterns and reports
// missing and ambiguous package errors. It also reports retractions and
// deprecations for resolved modules and modules needed to build named packages.
// It also adds a sum for each updated module in the build list if we had one
// before and didn't get one while loading packages.
//
// We skip missing-package errors earlier in the process, since we want to
// resolve pathSets ourselves, but at that point, we don't have enough context
// to log the package-import chains leading to each error.
func (r *resolver) checkPackagesAndRetractions(ctx context.Context, pkgPatterns []string) {
func (r *resolver) checkPackageProblems(ctx context.Context, pkgPatterns []string) {
defer base.ExitIfErrors()
// Build a list of modules to load retractions for. Start with versions
// selected based on command line queries.
//
// This is a subset of the build list. If the main module has a lot of
// dependencies, loading retractions for the entire build list would be slow.
relevantMods := make(map[module.Version]struct{})
// Gather information about modules we might want to load retractions and
// deprecations for. Loading this metadata requires at least one version
// lookup per module, and we don't want to load information that's neither
// relevant nor actionable.
type modFlags int
const (
resolved modFlags = 1 << iota // version resolved by 'go get'
named // explicitly named on command line or provides a named package
hasPkg // needed to build named packages
direct // provides a direct dependency of the main module
)
relevantMods := make(map[module.Version]modFlags)
for path, reason := range r.resolvedVersion {
relevantMods[module.Version{Path: path, Version: reason.version}] = struct{}{}
m := module.Version{Path: path, Version: reason.version}
relevantMods[m] |= resolved
}
// Reload packages, reporting errors for missing and ambiguous imports.
@ -1443,9 +1500,11 @@ func (r *resolver) checkPackagesAndRetractions(ctx context.Context, pkgPatterns
// LoadPackages will print errors (since it has more context) but will not
// exit, since we need to load retractions later.
pkgOpts := modload.PackageOpts{
LoadTests: *getT,
ResolveMissingImports: false,
AllowErrors: true,
VendorModulesInGOROOTSrc: true,
LoadTests: *getT,
ResolveMissingImports: false,
AllowErrors: true,
SilenceNoGoErrors: true,
}
matches, pkgs := modload.LoadPackages(ctx, pkgOpts, pkgPatterns...)
for _, m := range matches {
@ -1461,53 +1520,141 @@ func (r *resolver) checkPackagesAndRetractions(ctx context.Context, pkgPatterns
// associated with either the package or its test — ErrNoGo must
// indicate that none of those source files happen to apply in this
// configuration. If we are actually building the package (no -d
// flag), the compiler will report the problem; otherwise, assume that
// the user is going to build or test it in some other configuration
// and suppress the error.
// flag), we will report the problem then; otherwise, assume that the
// user is going to build or test this package in some other
// configuration and suppress the error.
continue
}
base.SetExitStatus(1)
if ambiguousErr := (*modload.AmbiguousImportError)(nil); errors.As(err, &ambiguousErr) {
for _, m := range ambiguousErr.Modules {
relevantMods[m] = struct{}{}
relevantMods[m] |= hasPkg
}
}
}
if m := modload.PackageModule(pkg); m.Path != "" {
relevantMods[m] = struct{}{}
relevantMods[m] |= hasPkg
}
}
for _, match := range matches {
for _, pkg := range match.Pkgs {
m := modload.PackageModule(pkg)
relevantMods[m] |= named
}
}
}
// Load and report retractions.
type retraction struct {
m module.Version
err error
}
retractions := make([]retraction, 0, len(relevantMods))
reqs := modload.LoadModFile(ctx)
for m := range relevantMods {
retractions = append(retractions, retraction{m: m})
if reqs.IsDirect(m.Path) {
relevantMods[m] |= direct
}
}
sort.Slice(retractions, func(i, j int) bool {
return retractions[i].m.Path < retractions[j].m.Path
})
for i := 0; i < len(retractions); i++ {
// Load retractions for modules mentioned on the command line and modules
// needed to build named packages. We care about retractions of indirect
// dependencies, since we might be able to upgrade away from them.
type modMessage struct {
m module.Version
message string
}
retractions := make([]modMessage, 0, len(relevantMods))
for m, flags := range relevantMods {
if flags&(resolved|named|hasPkg) != 0 {
retractions = append(retractions, modMessage{m: m})
}
}
sort.Slice(retractions, func(i, j int) bool { return retractions[i].m.Path < retractions[j].m.Path })
for i := range retractions {
i := i
r.work.Add(func() {
err := modload.CheckRetractions(ctx, retractions[i].m)
if retractErr := (*modload.ModuleRetractedError)(nil); errors.As(err, &retractErr) {
retractions[i].err = err
retractions[i].message = err.Error()
}
})
}
// Load deprecations for modules mentioned on the command line. Only load
// deprecations for indirect dependencies if they're also direct dependencies
// of the main module. Deprecations of purely indirect dependencies are
// not actionable.
deprecations := make([]modMessage, 0, len(relevantMods))
for m, flags := range relevantMods {
if flags&(resolved|named) != 0 || flags&(hasPkg|direct) == hasPkg|direct {
deprecations = append(deprecations, modMessage{m: m})
}
}
sort.Slice(deprecations, func(i, j int) bool { return deprecations[i].m.Path < deprecations[j].m.Path })
for i := range deprecations {
i := i
r.work.Add(func() {
deprecation, err := modload.CheckDeprecation(ctx, deprecations[i].m)
if err != nil || deprecation == "" {
return
}
deprecations[i].message = modload.ShortMessage(deprecation, "")
})
}
// Load sums for updated modules that had sums before. When we update a
// module, we may update another module in the build list that provides a
// package in 'all' that wasn't loaded as part of this 'go get' command.
// If we don't add a sum for that module, builds may fail later.
// Note that an incidentally updated package could still import packages
// from unknown modules or from modules in the build list that we didn't
// need previously. We can't handle that case without loading 'all'.
sumErrs := make([]error, len(r.buildList))
for i := range r.buildList {
i := i
m := r.buildList[i]
mActual := m
if mRepl := modload.Replacement(m); mRepl.Path != "" {
mActual = mRepl
}
old := module.Version{Path: m.Path, Version: r.initialVersion[m.Path]}
if old.Version == "" {
continue
}
oldActual := old
if oldRepl := modload.Replacement(old); oldRepl.Path != "" {
oldActual = oldRepl
}
if mActual == oldActual || mActual.Version == "" || !modfetch.HaveSum(oldActual) {
continue
}
r.work.Add(func() {
if _, err := modfetch.DownloadZip(ctx, mActual); err != nil {
verb := "upgraded"
if semver.Compare(m.Version, old.Version) < 0 {
verb = "downgraded"
}
replaced := ""
if mActual != m {
replaced = fmt.Sprintf(" (replaced by %s)", mActual)
}
err = fmt.Errorf("%s %s %s => %s%s: error finding sum for %s: %v", verb, m.Path, old.Version, m.Version, replaced, mActual, err)
sumErrs[i] = err
}
})
}
<-r.work.Idle()
// Report deprecations, then retractions, then errors fetching sums.
// Only errors fetching sums are hard errors.
for _, mm := range deprecations {
if mm.message != "" {
fmt.Fprintf(os.Stderr, "go: module %s is deprecated: %s\n", mm.m.Path, mm.message)
}
}
var retractPath string
for _, r := range retractions {
if r.err != nil {
fmt.Fprintf(os.Stderr, "go: warning: %v\n", r.err)
for _, mm := range retractions {
if mm.message != "" {
fmt.Fprintf(os.Stderr, "go: warning: %v\n", mm.message)
if retractPath == "" {
retractPath = r.m.Path
retractPath = mm.m.Path
} else {
retractPath = "<module>"
}
@ -1516,6 +1663,12 @@ func (r *resolver) checkPackagesAndRetractions(ctx context.Context, pkgPatterns
if retractPath != "" {
fmt.Fprintf(os.Stderr, "go: to switch to the latest unretracted version, run:\n\tgo get %s@latest\n", retractPath)
}
for _, err := range sumErrs {
if err != nil {
base.Errorf("go: %v", err)
}
}
base.ExitIfErrors()
}
// reportChanges logs version changes to os.Stderr.
@ -1614,11 +1767,10 @@ func (r *resolver) resolve(q *query, m module.Version) {
//
// If the additional modules conflict with the resolved versions, they will be
// downgraded to a non-conflicting version (possibly "none").
//
// If the resulting build list is the same as the one resulting from the last
// call to updateBuildList, updateBuildList returns with changed=false.
func (r *resolver) updateBuildList(ctx context.Context, additions []module.Version) (changed bool) {
if len(additions) == 0 && len(r.resolvedVersion) == r.buildListResolvedVersions {
return false
}
defer base.ExitIfErrors()
resolved := make([]module.Version, 0, len(r.resolvedVersion))
@ -1628,7 +1780,8 @@ func (r *resolver) updateBuildList(ctx context.Context, additions []module.Versi
}
}
if err := modload.EditBuildList(ctx, additions, resolved); err != nil {
changed, err := modload.EditBuildList(ctx, additions, resolved)
if err != nil {
var constraint *modload.ConstraintError
if !errors.As(err, &constraint) {
base.Errorf("go get: %v", err)
@ -1647,13 +1800,12 @@ func (r *resolver) updateBuildList(ctx context.Context, additions []module.Versi
}
return false
}
buildList := modload.LoadAllModules(ctx)
r.buildListResolvedVersions = len(r.resolvedVersion)
if reflect.DeepEqual(r.buildList, buildList) {
if !changed {
return false
}
r.buildList = buildList
const defaultGoVersion = ""
r.buildList = modload.LoadModGraph(ctx, defaultGoVersion).BuildList()
r.buildListVersion = make(map[string]string, len(r.buildList))
for _, m := range r.buildList {
r.buildListVersion[m.Path] = m.Version

33
libgo/go/cmd/go/internal/modinfo/info.go
View file

@ -10,19 +10,20 @@ import "time"
// and the fields are documented in the help text in ../list/list.go
type ModulePublic struct {
Path string `json:",omitempty"` // module path
Version string `json:",omitempty"` // module version
Versions []string `json:",omitempty"` // available module versions
Replace *ModulePublic `json:",omitempty"` // replaced by this module
Time *time.Time `json:",omitempty"` // time version was created
Update *ModulePublic `json:",omitempty"` // available update (with -u)
Main bool `json:",omitempty"` // is this the main module?
Indirect bool `json:",omitempty"` // module is only indirectly needed by main module
Dir string `json:",omitempty"` // directory holding local copy of files, if any
GoMod string `json:",omitempty"` // path to go.mod file describing module, if any
GoVersion string `json:",omitempty"` // go version used in module
Retracted []string `json:",omitempty"` // retraction information, if any (with -retracted or -u)
Error *ModuleError `json:",omitempty"` // error loading module
Path string `json:",omitempty"` // module path
Version string `json:",omitempty"` // module version
Versions []string `json:",omitempty"` // available module versions
Replace *ModulePublic `json:",omitempty"` // replaced by this module
Time *time.Time `json:",omitempty"` // time version was created
Update *ModulePublic `json:",omitempty"` // available update (with -u)
Main bool `json:",omitempty"` // is this the main module?
Indirect bool `json:",omitempty"` // module is only indirectly needed by main module
Dir string `json:",omitempty"` // directory holding local copy of files, if any
GoMod string `json:",omitempty"` // path to go.mod file describing module, if any
GoVersion string `json:",omitempty"` // go version used in module
Retracted []string `json:",omitempty"` // retraction information, if any (with -retracted or -u)
Deprecated string `json:",omitempty"` // deprecation message, if any (with -u)
Error *ModuleError `json:",omitempty"` // error loading module
}
type ModuleError struct {
@ -45,6 +46,9 @@ func (m *ModulePublic) String() string {
s += " [" + versionString(m.Update) + "]"
}
}
if m.Deprecated != "" {
s += " (deprecated)"
}
if m.Replace != nil {
s += " => " + m.Replace.Path
if m.Replace.Version != "" {
@ -53,6 +57,9 @@ func (m *ModulePublic) String() string {
s += " [" + versionString(m.Replace.Update) + "]"
}
}
if m.Replace.Deprecated != "" {
s += " (deprecated)"
}
}
return s
}

197
libgo/go/cmd/go/internal/modload/build.go
View file

@ -11,6 +11,7 @@ import (
"errors"
"fmt"
"internal/goroot"
"io/fs"
"os"
"path/filepath"
"strings"
@ -50,17 +51,17 @@ func findStandardImportPath(path string) string {
// a given package. If modules are not enabled or if the package is in the
// standard library or if the package was not successfully loaded with
// LoadPackages or ImportFromFiles, nil is returned.
func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic {
func PackageModuleInfo(ctx context.Context, pkgpath string) *modinfo.ModulePublic {
if isStandardImportPath(pkgpath) || !Enabled() {
return nil
}
m, ok := findModule(pkgpath)
m, ok := findModule(loaded, pkgpath)
if !ok {
return nil
}
fromBuildList := true
listRetracted := false
return moduleInfo(context.TODO(), m, fromBuildList, listRetracted)
rs := LoadModFile(ctx)
return moduleInfo(ctx, rs, m, 0)
}
func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic {
@ -68,26 +69,38 @@ func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic {
return nil
}
listRetracted := false
if i := strings.Index(path, "@"); i >= 0 {
m := module.Version{Path: path[:i], Version: path[i+1:]}
fromBuildList := false
return moduleInfo(ctx, m, fromBuildList, listRetracted)
return moduleInfo(ctx, nil, m, 0)
}
for _, m := range buildList {
if m.Path == path {
fromBuildList := true
return moduleInfo(ctx, m, fromBuildList, listRetracted)
rs := LoadModFile(ctx)
var (
v string
ok bool
)
if rs.depth == lazy {
v, ok = rs.rootSelected(path)
}
if !ok {
mg, err := rs.Graph(ctx)
if err != nil {
base.Fatalf("go: %v", err)
}
v = mg.Selected(path)
}
if v == "none" {
return &modinfo.ModulePublic{
Path: path,
Error: &modinfo.ModuleError{
Err: "module not in current build",
},
}
}
return &modinfo.ModulePublic{
Path: path,
Error: &modinfo.ModuleError{
Err: "module not in current build",
},
}
return moduleInfo(ctx, rs, module.Version{Path: path, Version: v}, 0)
}
// addUpdate fills in m.Update if an updated version is available.
@ -96,7 +109,26 @@ func addUpdate(ctx context.Context, m *modinfo.ModulePublic) {
return
}
if info, err := Query(ctx, m.Path, "upgrade", m.Version, CheckAllowed); err == nil && semver.Compare(info.Version, m.Version) > 0 {
info, err := Query(ctx, m.Path, "upgrade", m.Version, CheckAllowed)
var noVersionErr *NoMatchingVersionError
if errors.Is(err, fs.ErrNotExist) || errors.As(err, &noVersionErr) {
// Ignore "not found" and "no matching version" errors.
// This means the proxy has no matching version or no versions at all.
//
// We should report other errors though. An attacker that controls the
// network shouldn't be able to hide versions by interfering with
// the HTTPS connection. An attacker that controls the proxy may still
// hide versions, since the "list" and "latest" endpoints are not
// authenticated.
return
} else if err != nil {
if m.Error == nil {
m.Error = &modinfo.ModuleError{Err: err.Error()}
}
return
}
if semver.Compare(info.Version, m.Version) > 0 {
m.Update = &modinfo.ModulePublic{
Path: m.Path,
Version: info.Version,
@ -113,7 +145,11 @@ func addVersions(ctx context.Context, m *modinfo.ModulePublic, listRetracted boo
if listRetracted {
allowed = CheckExclusions
}
m.Versions, _ = versions(ctx, m.Path, allowed)
var err error
m.Versions, err = versions(ctx, m.Path, allowed)
if err != nil && m.Error == nil {
m.Error = &modinfo.ModuleError{Err: err.Error()}
}
}
// addRetraction fills in m.Retracted if the module was retracted by its author.
@ -124,31 +160,72 @@ func addRetraction(ctx context.Context, m *modinfo.ModulePublic) {
}
err := CheckRetractions(ctx, module.Version{Path: m.Path, Version: m.Version})
var rerr *ModuleRetractedError
if errors.As(err, &rerr) {
if len(rerr.Rationale) == 0 {
var noVersionErr *NoMatchingVersionError
var retractErr *ModuleRetractedError
if err == nil || errors.Is(err, fs.ErrNotExist) || errors.As(err, &noVersionErr) {
// Ignore "not found" and "no matching version" errors.
// This means the proxy has no matching version or no versions at all.
//
// We should report other errors though. An attacker that controls the
// network shouldn't be able to hide versions by interfering with
// the HTTPS connection. An attacker that controls the proxy may still
// hide versions, since the "list" and "latest" endpoints are not
// authenticated.
return
} else if errors.As(err, &retractErr) {
if len(retractErr.Rationale) == 0 {
m.Retracted = []string{"retracted by module author"}
} else {
m.Retracted = rerr.Rationale
m.Retracted = retractErr.Rationale
}
} else if err != nil && m.Error == nil {
} else if m.Error == nil {
m.Error = &modinfo.ModuleError{Err: err.Error()}
}
}
func moduleInfo(ctx context.Context, m module.Version, fromBuildList, listRetracted bool) *modinfo.ModulePublic {
// addDeprecation fills in m.Deprecated if the module was deprecated by its
// author. m.Error is set if there's an error loading deprecation information.
func addDeprecation(ctx context.Context, m *modinfo.ModulePublic) {
deprecation, err := CheckDeprecation(ctx, module.Version{Path: m.Path, Version: m.Version})
var noVersionErr *NoMatchingVersionError
if errors.Is(err, fs.ErrNotExist) || errors.As(err, &noVersionErr) {
// Ignore "not found" and "no matching version" errors.
// This means the proxy has no matching version or no versions at all.
//
// We should report other errors though. An attacker that controls the
// network shouldn't be able to hide versions by interfering with
// the HTTPS connection. An attacker that controls the proxy may still
// hide versions, since the "list" and "latest" endpoints are not
// authenticated.
return
}
if err != nil {
if m.Error == nil {
m.Error = &modinfo.ModuleError{Err: err.Error()}
}
return
}
m.Deprecated = deprecation
}
// moduleInfo returns information about module m, loaded from the requirements
// in rs (which may be nil to indicate that m was not loaded from a requirement
// graph).
func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode ListMode) *modinfo.ModulePublic {
if m == Target {
info := &modinfo.ModulePublic{
Path: m.Path,
Version: m.Version,
Main: true,
}
if v, ok := rawGoVersion.Load(Target); ok {
info.GoVersion = v.(string)
} else {
panic("internal error: GoVersion not set for main module")
}
if HasModRoot() {
info.Dir = ModRoot()
info.GoMod = ModFilePath()
if modFile.Go != nil {
info.GoVersion = modFile.Go.Version
}
}
return info
}
@ -156,7 +233,7 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList, listRetrac
info := &modinfo.ModulePublic{
Path: m.Path,
Version: m.Version,
Indirect: fromBuildList && loaded != nil && !loaded.direct[m.Path],
Indirect: rs != nil && !rs.direct[m.Path],
}
if v, ok := rawGoVersion.Load(m); ok {
info.GoVersion = v.(string)
@ -164,7 +241,10 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList, listRetrac
// completeFromModCache fills in the extra fields in m using the module cache.
completeFromModCache := func(m *modinfo.ModulePublic) {
mod := module.Version{Path: m.Path, Version: m.Version}
checksumOk := func(suffix string) bool {
return rs == nil || m.Version == "" || cfg.BuildMod == "mod" ||
modfetch.HaveSum(module.Version{Path: m.Path, Version: m.Version + suffix})
}
if m.Version != "" {
if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil {
@ -173,31 +253,40 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList, listRetrac
m.Version = q.Version
m.Time = &q.Time
}
}
mod := module.Version{Path: m.Path, Version: m.Version}
gomod, err := modfetch.CachePath(mod, "mod")
if err == nil {
if info, err := os.Stat(gomod); err == nil && info.Mode().IsRegular() {
m.GoMod = gomod
}
}
dir, err := modfetch.DownloadDir(mod)
if err == nil {
m.Dir = dir
}
if listRetracted {
addRetraction(ctx, m)
if m.GoVersion == "" && checksumOk("/go.mod") {
// Load the go.mod file to determine the Go version, since it hasn't
// already been populated from rawGoVersion.
if summary, err := rawGoModSummary(mod); err == nil && summary.goVersion != "" {
m.GoVersion = summary.goVersion
}
}
if m.GoVersion == "" {
if summary, err := rawGoModSummary(mod); err == nil && summary.goVersionV != "" {
m.GoVersion = summary.goVersionV[1:]
if m.Version != "" {
if checksumOk("/go.mod") {
gomod, err := modfetch.CachePath(mod, "mod")
if err == nil {
if info, err := os.Stat(gomod); err == nil && info.Mode().IsRegular() {
m.GoMod = gomod
}
}
}
if checksumOk("") {
dir, err := modfetch.DownloadDir(mod)
if err == nil {
m.Dir = dir
}
}
if mode&ListRetracted != 0 {
addRetraction(ctx, m)
}
}
}
if !fromBuildList {
if rs == nil {
// If this was an explicitly-versioned argument to 'go mod download' or
// 'go list -m', report the actual requested version, not its replacement.
completeFromModCache(info) // Will set m.Error in vendor mode.
@ -255,11 +344,11 @@ func PackageBuildInfo(path string, deps []string) string {
return ""
}
target := mustFindModule(path, path)
target := mustFindModule(loaded, path, path)
mdeps := make(map[module.Version]bool)
for _, dep := range deps {
if !isStandardImportPath(dep) {
mdeps[mustFindModule(path, dep)] = true
mdeps[mustFindModule(loaded, path, dep)] = true
}
}
var mods []module.Version
@ -298,8 +387,8 @@ func PackageBuildInfo(path string, deps []string) string {
//
// TODO(jayconrod): remove this. Callers should use findModule and return
// errors instead of relying on base.Fatalf.
func mustFindModule(target, path string) module.Version {
pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
func mustFindModule(ld *loader, target, path string) module.Version {
pkg, ok := ld.pkgCache.Get(path).(*loadPkg)
if ok {
if pkg.err != nil {
base.Fatalf("build %v: cannot load %v: %v", target, path, pkg.err)
@ -318,8 +407,8 @@ func mustFindModule(target, path string) module.Version {
// findModule searches for the module that contains the package at path.
// If the package was loaded, its containing module and true are returned.
// Otherwise, module.Version{} and false are returend.
func findModule(path string) (module.Version, bool) {
if pkg, ok := loaded.pkgCache.Get(path).(*loadPkg); ok {
func findModule(ld *loader, path string) (module.Version, bool) {
if pkg, ok := ld.pkgCache.Get(path).(*loadPkg); ok {
return pkg.mod, pkg.mod != module.Version{}
}
if path == "command-line-arguments" {

1226
libgo/go/cmd/go/internal/modload/buildlist.go
View file

File diff suppressed because it is too large Load diff

569
libgo/go/cmd/go/internal/modload/edit.go Normal file
View file

@ -0,0 +1,569 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package modload
import (
"cmd/go/internal/mvs"
"context"
"reflect"
"sort"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
)
// editRequirements returns an edited version of rs such that:
//
// 1. Each module version in mustSelect is selected.
//
// 2. Each module version in tryUpgrade is upgraded toward the indicated
// version as far as can be done without violating (1).
//
// 3. Each module version in rs.rootModules (or rs.graph, if rs.depth is eager)
// is downgraded from its original version only to the extent needed to
// satisfy (1), or upgraded only to the extent needed to satisfy (1) and
// (2).
//
// 4. No module is upgraded above the maximum version of its path found in the
// dependency graph of rs, the combined dependency graph of the versions in
// mustSelect, or the dependencies of each individual module version in
// tryUpgrade.
//
// Generally, the module versions in mustSelect are due to the module or a
// package within the module matching an explicit command line argument to 'go
// get', and the versions in tryUpgrade are transitive dependencies that are
// either being upgraded by 'go get -u' or being added to satisfy some
// otherwise-missing package import.
func editRequirements(ctx context.Context, rs *Requirements, tryUpgrade, mustSelect []module.Version) (edited *Requirements, changed bool, err error) {
limiter, err := limiterForEdit(ctx, rs, tryUpgrade, mustSelect)
if err != nil {
return rs, false, err
}
var conflicts []Conflict
for _, m := range mustSelect {
conflict, err := limiter.Select(m)
if err != nil {
return rs, false, err
}
if conflict.Path != "" {
conflicts = append(conflicts, Conflict{
Source: m,
Dep: conflict,
Constraint: module.Version{
Path: conflict.Path,
Version: limiter.max[conflict.Path],
},
})
}
}
if len(conflicts) > 0 {
return rs, false, &ConstraintError{Conflicts: conflicts}
}
mods, changed, err := selectPotentiallyImportedModules(ctx, limiter, rs, tryUpgrade)
if err != nil {
return rs, false, err
}
var roots []module.Version
if rs.depth == eager {
// In an eager module, modules that provide packages imported by the main
// module may either be explicit roots or implicit transitive dependencies.
// We promote the modules in mustSelect to be explicit requirements.
var rootPaths []string
for _, m := range mustSelect {
if m.Version != "none" && m.Path != Target.Path {
rootPaths = append(rootPaths, m.Path)
}
}
if !changed && len(rootPaths) == 0 {
// The build list hasn't changed and we have no new roots to add.
// We don't need to recompute the minimal roots for the module.
return rs, false, nil
}
for _, m := range mods {
if v, ok := rs.rootSelected(m.Path); ok && (v == m.Version || rs.direct[m.Path]) {
// m.Path was formerly a root, and either its version hasn't changed or
// we believe that it provides a package directly imported by a package
// or test in the main module. For now we'll assume that it is still
// relevant enough to remain a root. If we actually load all of the
// packages and tests in the main module (which we are not doing here),
// we can revise the explicit roots at that point.
rootPaths = append(rootPaths, m.Path)
}
}
roots, err = mvs.Req(Target, rootPaths, &mvsReqs{roots: mods})
if err != nil {
return nil, false, err
}
} else {
// In a lazy module, every module that provides a package imported by the
// main module must be retained as a root.
roots = mods
if !changed {
// Because the roots we just computed are unchanged, the entire graph must
// be the same as it was before. Save the original rs, since we have
// probably already loaded its requirement graph.
return rs, false, nil
}
}
// A module that is not even in the build list necessarily cannot provide
// any imported packages. Mark as direct only the direct modules that are
// still in the build list.
//
// TODO(bcmills): Would it make more sense to leave the direct map as-is
// but allow it to refer to modules that are no longer in the build list?
// That might complicate updateRoots, but it may be cleaner in other ways.
direct := make(map[string]bool, len(rs.direct))
for _, m := range roots {
if rs.direct[m.Path] {
direct[m.Path] = true
}
}
return newRequirements(rs.depth, roots, direct), changed, nil
}
// limiterForEdit returns a versionLimiter with its max versions set such that
// the max version for every module path in mustSelect is the version listed
// there, and the max version for every other module path is the maximum version
// of its path found in the dependency graph of rs, the combined dependency
// graph of the versions in mustSelect, or the dependencies of each individual
// module version in tryUpgrade.
func limiterForEdit(ctx context.Context, rs *Requirements, tryUpgrade, mustSelect []module.Version) (*versionLimiter, error) {
mg, err := rs.Graph(ctx)
if err != nil {
return nil, err
}
maxVersion := map[string]string{} // module path → version
restrictTo := func(m module.Version) {
v, ok := maxVersion[m.Path]
if !ok || cmpVersion(v, m.Version) > 0 {
maxVersion[m.Path] = m.Version
}
}
if rs.depth == eager {
// Eager go.mod files don't indicate which transitive dependencies are
// actually relevant to the main module, so we have to assume that any module
// that could have provided any package — that is, any module whose selected
// version was not "none" — may be relevant.
for _, m := range mg.BuildList() {
restrictTo(m)
}
} else {
// The go.mod file explicitly records every module that provides a package
// imported by the main module.
//
// If we need to downgrade an existing root or a new root found in
// tryUpgrade, we don't want to allow that downgrade to incidentally upgrade
// a module imported by the main module to some arbitrary version.
// However, we don't particularly care about arbitrary upgrades to modules
// that are (at best) only providing packages imported by tests of
// dependencies outside the main module.
for _, m := range rs.rootModules {
restrictTo(module.Version{
Path: m.Path,
Version: mg.Selected(m.Path),
})
}
}
if err := raiseLimitsForUpgrades(ctx, maxVersion, rs.depth, tryUpgrade, mustSelect); err != nil {
return nil, err
}
// The versions in mustSelect override whatever we would naively select —
// we will downgrade other modules as needed in order to meet them.
for _, m := range mustSelect {
restrictTo(m)
}
return newVersionLimiter(rs.depth, maxVersion), nil
}
// raiseLimitsForUpgrades increases the module versions in maxVersions to the
// versions that would be needed to allow each of the modules in tryUpgrade
// (individually) and all of the modules in mustSelect (simultaneously) to be
// added as roots.
//
// Versions not present in maxVersion are unrestricted, and it is assumed that
// they will not be promoted to root requirements (and thus will not contribute
// their own dependencies if the main module is lazy).
//
// These limits provide an upper bound on how far a module may be upgraded as
// part of an incidental downgrade, if downgrades are needed in order to select
// the versions in mustSelect.
func raiseLimitsForUpgrades(ctx context.Context, maxVersion map[string]string, depth modDepth, tryUpgrade []module.Version, mustSelect []module.Version) error {
// allow raises the limit for m.Path to at least m.Version.
// If m.Path was already unrestricted, it remains unrestricted.
allow := func(m module.Version) {
v, ok := maxVersion[m.Path]
if !ok {
return // m.Path is unrestricted.
}
if cmpVersion(v, m.Version) < 0 {
maxVersion[m.Path] = m.Version
}
}
var eagerUpgrades []module.Version
if depth == eager {
eagerUpgrades = tryUpgrade
} else {
for _, m := range tryUpgrade {
if m.Path == Target.Path {
// Target is already considered to be higher than any possible m, so we
// won't be upgrading to it anyway and there is no point scanning its
// dependencies.
continue
}
summary, err := goModSummary(m)
if err != nil {
return err
}
if summary.depth == eager {
// For efficiency, we'll load all of the eager upgrades as one big
// graph, rather than loading the (potentially-overlapping) subgraph for
// each upgrade individually.
eagerUpgrades = append(eagerUpgrades, m)
continue
}
for _, r := range summary.require {
allow(r)
}
}
}
if len(eagerUpgrades) > 0 {
// Compute the max versions for eager upgrades all together.
// Since these modules are eager, we'll end up scanning all of their
// transitive dependencies no matter which versions end up selected,
// and since we have a large dependency graph to scan we might get
// a significant benefit from not revisiting dependencies that are at
// common versions among multiple upgrades.
upgradeGraph, err := readModGraph(ctx, eager, eagerUpgrades)
if err != nil {
if go117LazyTODO {
// Compute the requirement path from a module path in tryUpgrade to the
// error, and the requirement path (if any) from rs.rootModules to the
// tryUpgrade module path. Return a *mvs.BuildListError showing the
// concatenation of the paths (with an upgrade in the middle).
}
return err
}
for _, r := range upgradeGraph.BuildList() {
// Upgrading to m would upgrade to r, and the caller requested that we
// try to upgrade to m, so it's ok to upgrade to r.
allow(r)
}
}
if len(mustSelect) > 0 {
mustGraph, err := readModGraph(ctx, depth, mustSelect)
if err != nil {
return err
}
for _, r := range mustGraph.BuildList() {
// Some module in mustSelect requires r, so we must allow at least r.Version
// unless it conflicts with an entry in mustSelect.
allow(r)
}
}
return nil
}
// selectPotentiallyImportedModules increases the limiter-selected version of
// every module in rs that potentially provides a package imported (directly or
// indirectly) by the main module, and every module in tryUpgrade, toward the
// highest version seen in rs or tryUpgrade, but not above the maximums enforced
// by the limiter.
//
// It returns the list of module versions selected by the limiter, sorted by
// path, along with a boolean indicating whether that list is different from the
// list of modules read from rs.
func selectPotentiallyImportedModules(ctx context.Context, limiter *versionLimiter, rs *Requirements, tryUpgrade []module.Version) (mods []module.Version, changed bool, err error) {
for _, m := range tryUpgrade {
if err := limiter.UpgradeToward(ctx, m); err != nil {
return nil, false, err
}
}
var initial []module.Version
if rs.depth == eager {
mg, err := rs.Graph(ctx)
if err != nil {
return nil, false, err
}
initial = mg.BuildList()[1:]
} else {
initial = rs.rootModules
}
for _, m := range initial {
if err := limiter.UpgradeToward(ctx, m); err != nil {
return nil, false, err
}
}
mods = make([]module.Version, 0, len(limiter.selected))
for path, v := range limiter.selected {
if v != "none" && path != Target.Path {
mods = append(mods, module.Version{Path: path, Version: v})
}
}
// We've identified acceptable versions for each of the modules, but those
// versions are not necessarily consistent with each other: one upgraded or
// downgraded module may require a higher (but still allowed) version of
// another. The lower version may require extraneous dependencies that aren't
// actually relevant, so we need to compute the actual selected versions.
mg, err := readModGraph(ctx, rs.depth, mods)
if err != nil {
return nil, false, err
}
mods = make([]module.Version, 0, len(limiter.selected))
for path, _ := range limiter.selected {
if path != Target.Path {
if v := mg.Selected(path); v != "none" {
mods = append(mods, module.Version{Path: path, Version: v})
}
}
}
module.Sort(mods)
changed = !reflect.DeepEqual(mods, initial)
return mods, changed, err
}
// A versionLimiter tracks the versions that may be selected for each module
// subject to constraints on the maximum versions of transitive dependencies.
type versionLimiter struct {
// depth is the depth at which the dependencies of the modules passed to
// Select and UpgradeToward are loaded.
depth modDepth
// max maps each module path to the maximum version that may be selected for
// that path.
//
// Paths with no entry are unrestricted, and we assume that they will not be
// promoted to root dependencies (so will not contribute dependencies if the
// main module is lazy).
max map[string]string
// selected maps each module path to a version of that path (if known) whose
// transitive dependencies do not violate any max version. The version kept
// is the highest one found during any call to UpgradeToward for the given
// module path.
//
// If a higher acceptable version is found during a call to UpgradeToward for
// some *other* module path, that does not update the selected version.
// Ignoring those versions keeps the downgrades computed for two modules
// together close to the individual downgrades that would be computed for each
// module in isolation. (The only way one module can affect another is if the
// final downgraded version of the one module explicitly requires a higher
// version of the other.)
//
// Version "none" of every module is always known not to violate any max
// version, so paths at version "none" are omitted.
selected map[string]string
// dqReason records whether and why each each encountered version is
// disqualified.
dqReason map[module.Version]dqState
// requiring maps each not-yet-disqualified module version to the versions
// that directly require it. If that version becomes disqualified, the
// disqualification will be propagated to all of the versions in the list.
requiring map[module.Version][]module.Version
}
// A dqState indicates whether and why a module version is “disqualified” from
// being used in a way that would incorporate its requirements.
//
// The zero dqState indicates that the module version is not known to be
// disqualified, either because it is ok or because we are currently traversing
// a cycle that includes it.
type dqState struct {
err error // if non-nil, disqualified because the requirements of the module could not be read
conflict module.Version // disqualified because the module (transitively) requires dep, which exceeds the maximum version constraint for its path
}
func (dq dqState) isDisqualified() bool {
return dq != dqState{}
}
// newVersionLimiter returns a versionLimiter that restricts the module paths
// that appear as keys in max.
//
// max maps each module path to its maximum version; paths that are not present
// in the map are unrestricted. The limiter assumes that unrestricted paths will
// not be promoted to root dependencies.
//
// If depth is lazy, then if a module passed to UpgradeToward or Select is
// itself lazy, its unrestricted dependencies are skipped when scanning
// requirements.
func newVersionLimiter(depth modDepth, max map[string]string) *versionLimiter {
return &versionLimiter{
depth: depth,
max: max,
selected: map[string]string{Target.Path: Target.Version},
dqReason: map[module.Version]dqState{},
requiring: map[module.Version][]module.Version{},
}
}
// UpgradeToward attempts to upgrade the selected version of m.Path as close as
// possible to m.Version without violating l's maximum version limits.
//
// If depth is lazy and m itself is lazy, the the dependencies of unrestricted
// dependencies of m will not be followed.
func (l *versionLimiter) UpgradeToward(ctx context.Context, m module.Version) error {
selected, ok := l.selected[m.Path]
if ok {
if cmpVersion(selected, m.Version) >= 0 {
// The selected version is already at least m, so no upgrade is needed.
return nil
}
} else {
selected = "none"
}
if l.check(m, l.depth).isDisqualified() {
candidates, err := versions(ctx, m.Path, CheckAllowed)
if err != nil {
// This is likely a transient error reaching the repository,
// rather than a permanent error with the retrieved version.
//
// TODO(golang.org/issue/31730, golang.org/issue/30134):
// decode what to do based on the actual error.
return err
}
// Skip to candidates < m.Version.
i := sort.Search(len(candidates), func(i int) bool {
return semver.Compare(candidates[i], m.Version) >= 0
})
candidates = candidates[:i]
for l.check(m, l.depth).isDisqualified() {
n := len(candidates)
if n == 0 || cmpVersion(selected, candidates[n-1]) >= 0 {
// We couldn't find a suitable candidate above the already-selected version.
// Retain that version unmodified.
return nil
}
m.Version, candidates = candidates[n-1], candidates[:n-1]
}
}
l.selected[m.Path] = m.Version
return nil
}
// Select attempts to set the selected version of m.Path to exactly m.Version.
func (l *versionLimiter) Select(m module.Version) (conflict module.Version, err error) {
dq := l.check(m, l.depth)
if !dq.isDisqualified() {
l.selected[m.Path] = m.Version
}
return dq.conflict, dq.err
}
// check determines whether m (or its transitive dependencies) would violate l's
// maximum version limits if added to the module requirement graph.
//
// If depth is lazy and m itself is lazy, then the dependencies of unrestricted
// dependencies of m will not be followed. If the lazy loading invariants hold
// for the main module up to this point, the packages in those modules are at
// best only imported by tests of dependencies that are themselves loaded from
// outside modules. Although we would like to keep 'go test all' as reproducible
// as is feasible, we don't want to retain test dependencies that are only
// marginally relevant at best.
func (l *versionLimiter) check(m module.Version, depth modDepth) dqState {
if m.Version == "none" || m == Target {
// version "none" has no requirements, and the dependencies of Target are
// tautological.
return dqState{}
}
if dq, seen := l.dqReason[m]; seen {
return dq
}
l.dqReason[m] = dqState{}
if max, ok := l.max[m.Path]; ok && cmpVersion(m.Version, max) > 0 {
return l.disqualify(m, dqState{conflict: m})
}
summary, err := goModSummary(m)
if err != nil {
// If we can't load the requirements, we couldn't load the go.mod file.
// There are a number of reasons this can happen, but this usually
// means an older version of the module had a missing or invalid
// go.mod file. For example, if example.com/mod released v2.0.0 before
// migrating to modules (v2.0.0+incompatible), then added a valid go.mod
// in v2.0.1, downgrading from v2.0.1 would cause this error.
//
// TODO(golang.org/issue/31730, golang.org/issue/30134): if the error
// is transient (we couldn't download go.mod), return the error from
// Downgrade. Currently, we can't tell what kind of error it is.
return l.disqualify(m, dqState{err: err})
}
if summary.depth == eager {
depth = eager
}
for _, r := range summary.require {
if depth == lazy {
if _, restricted := l.max[r.Path]; !restricted {
// r.Path is unrestricted, so we don't care at what version it is
// selected. We assume that r.Path will not become a root dependency, so
// since m is lazy, r's dependencies won't be followed.
continue
}
}
if dq := l.check(r, depth); dq.isDisqualified() {
return l.disqualify(m, dq)
}
// r and its dependencies are (perhaps provisionally) ok.
//
// However, if there are cycles in the requirement graph, we may have only
// checked a portion of the requirement graph so far, and r (and thus m) may
// yet be disqualified by some path we have not yet visited. Remember this edge
// so that we can disqualify m and its dependents if that occurs.
l.requiring[r] = append(l.requiring[r], m)
}
return dqState{}
}
// disqualify records that m (or one of its transitive dependencies)
// violates l's maximum version limits.
func (l *versionLimiter) disqualify(m module.Version, dq dqState) dqState {
if dq := l.dqReason[m]; dq.isDisqualified() {
return dq
}
l.dqReason[m] = dq
for _, p := range l.requiring[m] {
l.disqualify(p, dqState{conflict: m})
}
// Now that we have disqualified the modules that depend on m, we can forget
// about them — we won't need to disqualify them again.
delete(l.requiring, m)
return dq
}

Some files were not shown because too many files have changed in this diff Show more