FreeChainXenon/gcc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

libgo: update to Go1.10beta1

Browse source 
Update the Go library to the 1.10beta1 release.
    
    Requires a few changes to the compiler for modifications to the map
    runtime code, and to handle some nowritebarrier cases in the runtime.
    
    Reviewed-on: https://go-review.googlesource.com/86455

gotools/:
	* Makefile.am (go_cmd_vet_files): New variable.
	(go_cmd_buildid_files, go_cmd_test2json_files): New variables.
	(s-zdefaultcc): Change from constants to functions.
	(noinst_PROGRAMS): Add vet, buildid, and test2json.
	(cgo$(EXEEXT)): Link against $(LIBGOTOOL).
	(vet$(EXEEXT)): New target.
	(buildid$(EXEEXT)): New target.
	(test2json$(EXEEXT)): New target.
	(install-exec-local): Install all $(noinst_PROGRAMS).
	(uninstall-local): Uninstasll all $(noinst_PROGRAMS).
	(check-go-tool): Depend on $(noinst_PROGRAMS).  Copy down
	objabi.go.
	(check-runtime): Depend on $(noinst_PROGRAMS).
	(check-cgo-test, check-carchive-test): Likewise.
	(check-vet): New target.
	(check): Depend on check-vet.  Look at cmd_vet-testlog.
	(.PHONY): Add check-vet.
	* Makefile.in: Rebuild.

From-SVN: r256365
This commit is contained in:
Ian Lance Taylor 2018-01-09 01:23:08 +00:00 • committed by Ian Lance Taylor
parent 8799df67f2
commit 1a2f01efa6
1102 changed files with 73361 additions and 22970 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
gcc/go/gofrontend/MERGE
View file

@ -1,4 +1,4 @@
1319f36ccc65cf802b8e17ddd3d2da3ca6d82f4c
dbc0c7e4329aada2ae3554c20cfb8cfa48041213
The first line of this file holds the git revision number of the last
merge done from the gofrontend repository.

43
gcc/go/gofrontend/expressions.cc
View file

@ -7483,6 +7483,7 @@ Builtin_call_expression::lower_make(Statement_inserter* inserter)
return Expression::make_error(this->location());
}
len_arg = Expression::make_integer_ul(0, NULL, loc);
len_small = true;
}
else
{
@ -7551,9 +7552,23 @@ Builtin_call_expression::lower_make(Statement_inserter* inserter)
else if (is_map)
{
Expression* type_arg = Expression::make_type_descriptor(type, type_loc);
call = Runtime::make_call(Runtime::MAKEMAP, loc, 4, type_arg, len_arg,
Expression::make_nil(loc),
Expression::make_nil(loc));
if (!len_small)
call = Runtime::make_call(Runtime::MAKEMAP64, loc, 3, type_arg,
len_arg,
Expression::make_nil(loc));
else
{
Numeric_constant nclen;
unsigned long vlen;
if (len_arg->numeric_constant_value(&nclen)
&& nclen.to_unsigned_long(&vlen) == Numeric_constant::NC_UL_VALID
&& vlen <= Map_type::bucket_size)
call = Runtime::make_call(Runtime::MAKEMAP_SMALL, loc, 0);
else
call = Runtime::make_call(Runtime::MAKEMAP, loc, 3, type_arg,
len_arg,
Expression::make_nil(loc));
}
}
else if (is_chan)
{
@ -9503,14 +9518,8 @@ Call_expression::do_lower(Gogo* gogo, Named_object* function,
// could implement them in normal code, but then we would have to
// explicitly unwind the stack. These functions are intended to be
// efficient. Note that this technique obviously only works for
// direct calls, but that is the only way they are used. The actual
// argument to these functions is always the address of a parameter;
// we don't need that for the GCC builtin functions, so we just
// ignore it.
if (gogo->compiling_runtime()
&& this->args_ != NULL
&& this->args_->size() == 1
&& gogo->package_name() == "runtime")
// direct calls, but that is the only way they are used.
if (gogo->compiling_runtime() && gogo->package_name() == "runtime")
{
Func_expression* fe = this->fn_->func_expression();
if (fe != NULL
@ -9518,15 +9527,21 @@ Call_expression::do_lower(Gogo* gogo, Named_object* function,
&& fe->named_object()->package() == NULL)
{
std::string n = Gogo::unpack_hidden_name(fe->named_object()->name());
if (n == "getcallerpc")
if ((this->args_ == NULL || this->args_->size() == 0)
&& n == "getcallerpc")
{
static Named_object* builtin_return_address;
return this->lower_to_builtin(&builtin_return_address,
"__builtin_return_address",
0);
}
else if (n == "getcallersp")
else if (this->args_ != NULL
&& this->args_->size() == 1
&& n == "getcallersp")
{
// The actual argument to getcallersp is always the
// address of a parameter; we don't need that for the
// GCC builtin function, so we just ignore it.
static Named_object* builtin_frame_address;
return this->lower_to_builtin(&builtin_frame_address,
"__builtin_frame_address",
@ -10027,7 +10042,7 @@ Call_expression::do_check_types(Gogo*)
}
const Typed_identifier_list* parameters = fntype->parameters();
if (this->args_ == NULL)
if (this->args_ == NULL || this->args_->size() == 0)
{
if (parameters != NULL && !parameters->empty())
this->report_error(_("not enough arguments"));

11
gcc/go/gofrontend/runtime.def
View file

@ -91,9 +91,14 @@ DEF_GO_RUNTIME(MAKESLICE64, "runtime.makeslice64", P3(TYPE, INT64, INT64),
R1(SLICE))
// Make a map.
DEF_GO_RUNTIME(MAKEMAP, "runtime.makemap", P4(TYPE, INT64, POINTER, POINTER),
R1(MAP))
// Make a map with a hint and an (optional, unused) map structure.
DEF_GO_RUNTIME(MAKEMAP, "runtime.makemap", P3(TYPE, INT, POINTER),
R1(MAP))
DEF_GO_RUNTIME(MAKEMAP64, "runtime.makemap64", P3(TYPE, INT64, POINTER),
R1(MAP))
// Make a map with no hint, or a small constant hint.
DEF_GO_RUNTIME(MAKEMAP_SMALL, "runtime.makemap_small", P0(), R1(MAP))
// Build a map from a composite literal.
DEF_GO_RUNTIME(CONSTRUCT_MAP, "__go_construct_map",

20
gcc/go/gofrontend/types.cc
View file

@ -7830,7 +7830,7 @@ Map_type::do_get_backend(Gogo* gogo)
bfields[7].btype = uintptr_type->get_backend(gogo);
bfields[7].location = bloc;
bfields[8].name = "overflow";
bfields[8].name = "extra";
bfields[8].btype = bpvt;
bfields[8].location = bloc;
@ -8144,21 +8144,23 @@ Map_type::hmap_type(Type* bucket_type)
Type* int_type = Type::lookup_integer_type("int");
Type* uint8_type = Type::lookup_integer_type("uint8");
Type* uint16_type = Type::lookup_integer_type("uint16");
Type* uint32_type = Type::lookup_integer_type("uint32");
Type* uintptr_type = Type::lookup_integer_type("uintptr");
Type* void_ptr_type = Type::make_pointer_type(Type::make_void_type());
Type* ptr_bucket_type = Type::make_pointer_type(bucket_type);
Struct_type* ret = make_builtin_struct_type(8,
Struct_type* ret = make_builtin_struct_type(9,
"count", int_type,
"flags", uint8_type,
"B", uint8_type,
"noverflow", uint16_type,
"hash0", uint32_type,
"buckets", ptr_bucket_type,
"oldbuckets", ptr_bucket_type,
"nevacuate", uintptr_type,
"overflow", void_ptr_type);
"extra", void_ptr_type);
ret->set_is_struct_incomparable();
this->hmap_type_ = ret;
return ret;
@ -8191,18 +8193,22 @@ Map_type::hiter_type(Gogo* gogo)
Type* hmap_type = this->hmap_type(bucket_type);
Type* hmap_ptr_type = Type::make_pointer_type(hmap_type);
Type* void_ptr_type = Type::make_pointer_type(Type::make_void_type());
Type* bool_type = Type::lookup_bool_type();
Struct_type* ret = make_builtin_struct_type(12,
Struct_type* ret = make_builtin_struct_type(15,
"key", key_ptr_type,
"val", val_ptr_type,
"t", uint8_ptr_type,
"h", hmap_ptr_type,
"buckets", bucket_ptr_type,
"bptr", bucket_ptr_type,
"overflow0", void_ptr_type,
"overflow1", void_ptr_type,
"overflow", void_ptr_type,
"oldoverflow", void_ptr_type,
"startBucket", uintptr_type,
"stuff", uintptr_type,
"offset", uint8_type,
"wrapped", bool_type,
"B", uint8_type,
"i", uint8_type,
"bucket", uintptr_type,
"checkBucket", uintptr_type);
ret->set_is_struct_incomparable();

4
gcc/go/gofrontend/types.h
View file

@ -2826,6 +2826,9 @@ class Map_type : public Type
static Type*
make_map_type_descriptor_type();
// This must be in sync with libgo/go/runtime/hashmap.go.
static const int bucket_size = 8;
protected:
int
do_traverse(Traverse*);
@ -2867,7 +2870,6 @@ class Map_type : public Type
private:
// These must be in sync with libgo/go/runtime/hashmap.go.
static const int bucket_size = 8;
static const int max_key_size = 128;
static const int max_val_size = 128;
static const int max_zero_size = 1024;

49
gcc/go/gofrontend/wb.cc
View file

@ -331,6 +331,25 @@ Gogo::assign_needs_write_barrier(Expression* lhs)
if (!lhs->type()->has_pointer())
return false;
// An assignment to a field is handled like an assignment to the
// struct.
while (true)
{
// Nothing to do for a type that can not be in the heap, or a
// pointer to a type that can not be in the heap. We check this
// at each level of a struct.
if (!lhs->type()->in_heap())
return false;
if (lhs->type()->points_to() != NULL
&& !lhs->type()->points_to()->in_heap())
return false;
Field_reference_expression* fre = lhs->field_reference_expression();
if (fre == NULL)
break;
lhs = fre->expr();
}
// Nothing to do for an assignment to a temporary.
if (lhs->temporary_reference_expression() != NULL)
return false;
@ -359,12 +378,30 @@ Gogo::assign_needs_write_barrier(Expression* lhs)
}
}
// Nothing to do for a type that can not be in the heap, or a
// pointer to a type that can not be in the heap.
if (!lhs->type()->in_heap())
return false;
if (lhs->type()->points_to() != NULL && !lhs->type()->points_to()->in_heap())
return false;
// For a struct assignment, we don't need a write barrier if all the
// pointer types can not be in the heap.
Struct_type* st = lhs->type()->struct_type();
if (st != NULL)
{
bool in_heap = false;
const Struct_field_list* fields = st->fields();
for (Struct_field_list::const_iterator p = fields->begin();
p != fields->end();
p++)
{
Type* ft = p->type();
if (!ft->has_pointer())
continue;
if (!ft->in_heap())
continue;
if (ft->points_to() != NULL && !ft->points_to()->in_heap())
continue;
in_heap = true;
break;
}
if (!in_heap)
return false;
}
// Write barrier needed in other cases.
return true;

21
gotools/ChangeLog
View file

@ -1,3 +1,24 @@
2018-01-08 Ian Lance Taylor <iant@golang.org>
* Makefile.am (go_cmd_vet_files): New variable.
(go_cmd_buildid_files, go_cmd_test2json_files): New variables.
(s-zdefaultcc): Change from constants to functions.
(noinst_PROGRAMS): Add vet, buildid, and test2json.
(cgo$(EXEEXT)): Link against $(LIBGOTOOL).
(vet$(EXEEXT)): New target.
(buildid$(EXEEXT)): New target.
(test2json$(EXEEXT)): New target.
(install-exec-local): Install all $(noinst_PROGRAMS).
(uninstall-local): Uninstasll all $(noinst_PROGRAMS).
(check-go-tool): Depend on $(noinst_PROGRAMS). Copy down
objabi.go.
(check-runtime): Depend on $(noinst_PROGRAMS).
(check-cgo-test, check-carchive-test): Likewise.
(check-vet): New target.
(check): Depend on check-vet. Look at cmd_vet-testlog.
(.PHONY): Add check-vet.
* Makefile.in: Rebuild.
2017-10-25 Ian Lance Taylor <iant@golang.org>
* Makefile.am (check-go-tool): Output colon after ${fl}.

95
gotools/Makefile.am
View file

@ -69,6 +69,40 @@ go_cmd_cgo_files = \
$(cmdsrcdir)/cgo/out.go \
$(cmdsrcdir)/cgo/util.go
go_cmd_vet_files = \
$(cmdsrcdir)/vet/asmdecl.go \
$(cmdsrcdir)/vet/assign.go \
$(cmdsrcdir)/vet/atomic.go \
$(cmdsrcdir)/vet/bool.go \
$(cmdsrcdir)/vet/buildtag.go \
$(cmdsrcdir)/vet/cgo.go \
$(cmdsrcdir)/vet/composite.go \
$(cmdsrcdir)/vet/copylock.go \
$(cmdsrcdir)/vet/deadcode.go \
$(cmdsrcdir)/vet/dead.go \
$(cmdsrcdir)/vet/doc.go \
$(cmdsrcdir)/vet/httpresponse.go \
$(cmdsrcdir)/vet/lostcancel.go \
$(cmdsrcdir)/vet/main.go \
$(cmdsrcdir)/vet/method.go \
$(cmdsrcdir)/vet/nilfunc.go \
$(cmdsrcdir)/vet/print.go \
$(cmdsrcdir)/vet/rangeloop.go \
$(cmdsrcdir)/vet/shadow.go \
$(cmdsrcdir)/vet/shift.go \
$(cmdsrcdir)/vet/structtag.go \
$(cmdsrcdir)/vet/tests.go \
$(cmdsrcdir)/vet/types.go \
$(cmdsrcdir)/vet/unsafeptr.go \
$(cmdsrcdir)/vet/unused.go
go_cmd_buildid_files = \
$(cmdsrcdir)/buildid/buildid.go \
$(cmdsrcdir)/buildid/doc.go
go_cmd_test2json_files = \
$(cmdsrcdir)/test2json/main.go
GCCGO_INSTALL_NAME := $(shell echo gccgo|sed '$(program_transform_name)')
GCC_INSTALL_NAME := $(shell echo gcc|sed '$(program_transform_name)')
GXX_INSTALL_NAME := $(shell echo g++|sed '$(program_transform_name)')
@ -76,9 +110,9 @@ GXX_INSTALL_NAME := $(shell echo g++|sed '$(program_transform_name)')
zdefaultcc.go: s-zdefaultcc; @true
s-zdefaultcc: Makefile
echo 'package main' > zdefaultcc.go.tmp
echo 'const defaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'const defaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'const defaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'func defaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'func defaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'func defaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'const defaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
$(STAMP) $@
@ -97,23 +131,33 @@ if NATIVE
# and install them as regular programs.
bin_PROGRAMS = go$(EXEEXT) gofmt$(EXEEXT)
noinst_PROGRAMS = cgo$(EXEEXT)
noinst_PROGRAMS = cgo$(EXEEXT) vet$(EXEEXT) buildid$(EXEEXT) test2json$(EXEEXT)
man_MANS = go.1 gofmt.1
go$(EXEEXT): $(go_cmd_go_files) $(LIBGOTOOL) $(LIBGODEP)
$(GOLINK) $(go_cmd_go_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
gofmt$(EXEEXT): $(go_cmd_gofmt_files) $(LIBGODEP)
$(GOLINK) $(go_cmd_gofmt_files) $(LIBS) $(NET_LIBS)
cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGODEP)
$(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBS) $(NET_LIBS)
cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBGODEP)
$(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
vet$(EXEEXT): $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBGODEP)
$(GOLINK) $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
buildid$(EXEEXT): $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBGODEP)
$(GOLINK) $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
test2json$(EXEEXT): $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBGODEP)
$(GOLINK) $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
install-exec-local: cgo$(EXEEXT)
install-exec-local: $(noinst_PROGRAMS)
$(MKDIR_P) $(DESTDIR)$(libexecsubdir)
rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
$(INSTALL_PROGRAM) cgo$(exeext) $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
for f in $(noinst_PROGRAMS); do \
rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
$(INSTALL_PROGRAM) $$f $(DESTDIR)$(libexecsubdir)/$$f; \
done
uninstall-local:
rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
for f in $(noinst_PROGRAMS); do \
rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
done
GOTESTFLAGS =
@ -177,8 +221,8 @@ CHECK_ENV = \
# It assumes that abs_libgodir is set.
ECHO_ENV = PATH=`echo $(abs_builddir):$${PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'` GCCGO='$(abs_builddir)/check-gccgo' CC='$(abs_builddir)/check-gcc' GCCGOTOOLDIR='$(abs_builddir)' GO_TESTING_GOTOOLS=yes LD_LIBRARY_PATH=`echo $${abs_libgodir}/.libs:$${LD_LIBRARY_PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'` GOROOT=`echo $${abs_libgodir}`
# check-go-tools runs `go test cmd/go` in our environment.
check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
# check-go-tool runs `go test cmd/go` in our environment.
check-go-tool: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
rm -rf check-go-dir cmd_go-testlog
$(MKDIR_P) check-go-dir/src/cmd/go
cp $(cmdsrcdir)/go/*.go check-go-dir/src/cmd/go/
@ -187,6 +231,7 @@ check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
cp $(libgodir)/zdefaultcc.go check-go-dir/src/cmd/go/internal/cfg/
cp -r $(cmdsrcdir)/go/testdata check-go-dir/src/cmd/go/
cp -r $(cmdsrcdir)/internal check-go-dir/src/cmd/
cp $(libgodir)/objabi.go check-go-dir/src/cmd/internal/objabi/
@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
abs_checkdir=`cd check-go-dir && $(PWD_COMMAND)`; \
echo "cd check-go-dir/src/cmd/go && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_go-testlog
@ -200,7 +245,7 @@ check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
# The runtime package is also tested as part of libgo,
# but the runtime tests use the go tool heavily, so testing
# here too will catch more problems.
check-runtime: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
check-runtime: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
rm -rf check-runtime-dir runtime-testlog
$(MKDIR_P) check-runtime-dir
@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
@ -219,7 +264,7 @@ check-runtime: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
grep '^--- ' runtime-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
# check-cgo-test runs `go test misc/cgo/test` in our environment.
check-cgo-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
check-cgo-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
rm -rf cgo-test-dir cgo-testlog
$(MKDIR_P) cgo-test-dir/misc/cgo
cp -r $(libgomiscdir)/cgo/test cgo-test-dir/misc/cgo/
@ -233,7 +278,7 @@ check-cgo-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
# check-carchive-test runs `go test misc/cgo/testcarchive/carchive_test.go`
# in our environment.
check-carchive-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
check-carchive-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
rm -rf carchive-test-dir carchive-testlog
$(MKDIR_P) carchive-test-dir/misc/cgo
cp -r $(libgomiscdir)/cgo/testcarchive carchive-test-dir/misc/cgo/
@ -245,11 +290,25 @@ check-carchive-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
(cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl}: go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog
grep '^--- ' carchive-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
# check-vet runs `go test cmd/vet` in our environment.
check-vet: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
rm -rf check-vet-dir cmd_vet-testlog
$(MKDIR_P) check-vet-dir/src/cmd
cp -r $(cmdsrcdir)/vet check-vet-dir/src/cmd/
@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
abs_checkdir=`cd check-vet-dir && $(PWD_COMMAND)`; \
echo "cd check-vet-dir/src/cmd/vet && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_vet-testlog
$(CHECK_ENV) \
GOPATH=`cd check-vet-dir && $(PWD_COMMAND)`; \
export GOPATH; \
(cd check-vet-dir/src/cmd/vet && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_vet-testlog 2>&1 || echo "--- $${fl}: go test cmd/vet (0.00s)" >> cmd_vet-testlog
grep '^--- ' cmd_vet-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
# The check targets runs the tests and assembles the output files.
check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test
check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
@mv gotools.head gotools.sum
@cp gotools.sum gotools.log
@for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog; do \
@for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog cmd_vet-testlog; do \
testname=`echo $${file} | sed -e 's/-testlog//' -e 's|_|/|'`; \
echo "Running $${testname}" >> gotools.sum; \
echo "Running $${testname}" >> gotools.log; \
@ -275,7 +334,7 @@ check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test
@echo "runtest completed at `date`" >> gotools.log
@if grep '^FAIL' gotools.sum >/dev/null 2>&1; then exit 1; fi
.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test
.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
else

118
gotools/Makefile.in
View file

@ -88,6 +88,9 @@ CONFIG_CLEAN_FILES =
CONFIG_CLEAN_VPATH_FILES =
am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(man1dir)"
PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS)
buildid_SOURCES = buildid.c
buildid_OBJECTS = buildid.$(OBJEXT)
buildid_LDADD = $(LDADD)
cgo_SOURCES = cgo.c
cgo_OBJECTS = cgo.$(OBJEXT)
cgo_LDADD = $(LDADD)
@ -97,6 +100,12 @@ go_LDADD = $(LDADD)
gofmt_SOURCES = gofmt.c
gofmt_OBJECTS = gofmt.$(OBJEXT)
gofmt_LDADD = $(LDADD)
test2json_SOURCES = test2json.c
test2json_OBJECTS = test2json.$(OBJEXT)
test2json_LDADD = $(LDADD)
vet_SOURCES = vet.c
vet_OBJECTS = vet.$(OBJEXT)
vet_LDADD = $(LDADD)
DEFAULT_INCLUDES = -I.@am__isrc@
depcomp = $(SHELL) $(top_srcdir)/../depcomp
am__depfiles_maybe = depfiles
@ -105,7 +114,7 @@ COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
CCLD = $(CC)
LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
SOURCES = cgo.c go.c gofmt.c
SOURCES = buildid.c cgo.c go.c gofmt.c test2json.c vet.c
am__can_run_installinfo = \
case $$AM_UPDATE_INFO_DIR in \
n|no|NO) false;; \
@ -288,6 +297,40 @@ go_cmd_cgo_files = \
$(cmdsrcdir)/cgo/out.go \
$(cmdsrcdir)/cgo/util.go
go_cmd_vet_files = \
$(cmdsrcdir)/vet/asmdecl.go \
$(cmdsrcdir)/vet/assign.go \
$(cmdsrcdir)/vet/atomic.go \
$(cmdsrcdir)/vet/bool.go \
$(cmdsrcdir)/vet/buildtag.go \
$(cmdsrcdir)/vet/cgo.go \
$(cmdsrcdir)/vet/composite.go \
$(cmdsrcdir)/vet/copylock.go \
$(cmdsrcdir)/vet/deadcode.go \
$(cmdsrcdir)/vet/dead.go \
$(cmdsrcdir)/vet/doc.go \
$(cmdsrcdir)/vet/httpresponse.go \
$(cmdsrcdir)/vet/lostcancel.go \
$(cmdsrcdir)/vet/main.go \
$(cmdsrcdir)/vet/method.go \
$(cmdsrcdir)/vet/nilfunc.go \
$(cmdsrcdir)/vet/print.go \
$(cmdsrcdir)/vet/rangeloop.go \
$(cmdsrcdir)/vet/shadow.go \
$(cmdsrcdir)/vet/shift.go \
$(cmdsrcdir)/vet/structtag.go \
$(cmdsrcdir)/vet/tests.go \
$(cmdsrcdir)/vet/types.go \
$(cmdsrcdir)/vet/unsafeptr.go \
$(cmdsrcdir)/vet/unused.go
go_cmd_buildid_files = \
$(cmdsrcdir)/buildid/buildid.go \
$(cmdsrcdir)/buildid/doc.go
go_cmd_test2json_files = \
$(cmdsrcdir)/test2json/main.go
GCCGO_INSTALL_NAME := $(shell echo gccgo|sed '$(program_transform_name)')
GCC_INSTALL_NAME := $(shell echo gcc|sed '$(program_transform_name)')
GXX_INSTALL_NAME := $(shell echo g++|sed '$(program_transform_name)')
@ -300,7 +343,7 @@ MOSTLYCLEANFILES = \
# For a native build we build the programs using the newly built libgo
# and install them as regular programs.
@NATIVE_TRUE@bin_PROGRAMS = go$(EXEEXT) gofmt$(EXEEXT)
@NATIVE_TRUE@noinst_PROGRAMS = cgo$(EXEEXT)
@NATIVE_TRUE@noinst_PROGRAMS = cgo$(EXEEXT) vet$(EXEEXT) buildid$(EXEEXT) test2json$(EXEEXT)
@NATIVE_TRUE@man_MANS = go.1 gofmt.1
@NATIVE_TRUE@GOTESTFLAGS =
@ -411,6 +454,9 @@ clean-binPROGRAMS:
clean-noinstPROGRAMS:
-test -z "$(noinst_PROGRAMS)" || rm -f $(noinst_PROGRAMS)
@NATIVE_FALSE@buildid$(EXEEXT): $(buildid_OBJECTS) $(buildid_DEPENDENCIES) $(EXTRA_buildid_DEPENDENCIES)
@NATIVE_FALSE@ @rm -f buildid$(EXEEXT)
@NATIVE_FALSE@ $(LINK) $(buildid_OBJECTS) $(buildid_LDADD) $(LIBS)
@NATIVE_FALSE@cgo$(EXEEXT): $(cgo_OBJECTS) $(cgo_DEPENDENCIES) $(EXTRA_cgo_DEPENDENCIES)
@NATIVE_FALSE@ @rm -f cgo$(EXEEXT)
@NATIVE_FALSE@ $(LINK) $(cgo_OBJECTS) $(cgo_LDADD) $(LIBS)
@ -420,6 +466,12 @@ clean-noinstPROGRAMS:
@NATIVE_FALSE@gofmt$(EXEEXT): $(gofmt_OBJECTS) $(gofmt_DEPENDENCIES) $(EXTRA_gofmt_DEPENDENCIES)
@NATIVE_FALSE@ @rm -f gofmt$(EXEEXT)
@NATIVE_FALSE@ $(LINK) $(gofmt_OBJECTS) $(gofmt_LDADD) $(LIBS)
@NATIVE_FALSE@test2json$(EXEEXT): $(test2json_OBJECTS) $(test2json_DEPENDENCIES) $(EXTRA_test2json_DEPENDENCIES)
@NATIVE_FALSE@ @rm -f test2json$(EXEEXT)
@NATIVE_FALSE@ $(LINK) $(test2json_OBJECTS) $(test2json_LDADD) $(LIBS)
@NATIVE_FALSE@vet$(EXEEXT): $(vet_OBJECTS) $(vet_DEPENDENCIES) $(EXTRA_vet_DEPENDENCIES)
@NATIVE_FALSE@ @rm -f vet$(EXEEXT)
@NATIVE_FALSE@ $(LINK) $(vet_OBJECTS) $(vet_LDADD) $(LIBS)
mostlyclean-compile:
-rm -f *.$(OBJEXT)
@ -427,9 +479,12 @@ mostlyclean-compile:
distclean-compile:
-rm -f *.tab.c
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/buildid.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cgo.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/go.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gofmt.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test2json.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/vet.Po@am__quote@
.c.o:
@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
@ -676,9 +731,9 @@ uninstall-man: uninstall-man1
zdefaultcc.go: s-zdefaultcc; @true
s-zdefaultcc: Makefile
echo 'package main' > zdefaultcc.go.tmp
echo 'const defaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'const defaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'const defaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'func defaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'func defaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'func defaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'const defaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
$(STAMP) $@
@ -690,16 +745,26 @@ mostlyclean-local:
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_go_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
@NATIVE_TRUE@gofmt$(EXEEXT): $(go_cmd_gofmt_files) $(LIBGODEP)
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_gofmt_files) $(LIBS) $(NET_LIBS)
@NATIVE_TRUE@cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGODEP)
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBS) $(NET_LIBS)
@NATIVE_TRUE@cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBGODEP)
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
@NATIVE_TRUE@vet$(EXEEXT): $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBGODEP)
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
@NATIVE_TRUE@buildid$(EXEEXT): $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBGODEP)
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
@NATIVE_TRUE@test2json$(EXEEXT): $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBGODEP)
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
@NATIVE_TRUE@install-exec-local: cgo$(EXEEXT)
@NATIVE_TRUE@install-exec-local: $(noinst_PROGRAMS)
@NATIVE_TRUE@ $(MKDIR_P) $(DESTDIR)$(libexecsubdir)
@NATIVE_TRUE@ rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
@NATIVE_TRUE@ $(INSTALL_PROGRAM) cgo$(exeext) $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
@NATIVE_TRUE@ for f in $(noinst_PROGRAMS); do \
@NATIVE_TRUE@ rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
@NATIVE_TRUE@ $(INSTALL_PROGRAM) $$f $(DESTDIR)$(libexecsubdir)/$$f; \
@NATIVE_TRUE@ done
@NATIVE_TRUE@uninstall-local:
@NATIVE_TRUE@ rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
@NATIVE_TRUE@ for f in $(noinst_PROGRAMS); do \
@NATIVE_TRUE@ rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
@NATIVE_TRUE@ done
# Run tests using the go tool, and frob the output to look like that
# generated by DejaGNU. The main output of this is two files:
@ -735,8 +800,8 @@ mostlyclean-local:
@NATIVE_TRUE@ chmod +x $@.tmp
@NATIVE_TRUE@ mv -f $@.tmp $@
# check-go-tools runs `go test cmd/go` in our environment.
@NATIVE_TRUE@check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
# check-go-tool runs `go test cmd/go` in our environment.
@NATIVE_TRUE@check-go-tool: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
@NATIVE_TRUE@ rm -rf check-go-dir cmd_go-testlog
@NATIVE_TRUE@ $(MKDIR_P) check-go-dir/src/cmd/go
@NATIVE_TRUE@ cp $(cmdsrcdir)/go/*.go check-go-dir/src/cmd/go/
@ -745,6 +810,7 @@ mostlyclean-local:
@NATIVE_TRUE@ cp $(libgodir)/zdefaultcc.go check-go-dir/src/cmd/go/internal/cfg/
@NATIVE_TRUE@ cp -r $(cmdsrcdir)/go/testdata check-go-dir/src/cmd/go/
@NATIVE_TRUE@ cp -r $(cmdsrcdir)/internal check-go-dir/src/cmd/
@NATIVE_TRUE@ cp $(libgodir)/objabi.go check-go-dir/src/cmd/internal/objabi/
@NATIVE_TRUE@ @abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
@NATIVE_TRUE@ abs_checkdir=`cd check-go-dir && $(PWD_COMMAND)`; \
@NATIVE_TRUE@ echo "cd check-go-dir/src/cmd/go && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_go-testlog
@ -758,7 +824,7 @@ mostlyclean-local:
# The runtime package is also tested as part of libgo,
# but the runtime tests use the go tool heavily, so testing
# here too will catch more problems.
@NATIVE_TRUE@check-runtime: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
@NATIVE_TRUE@check-runtime: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
@NATIVE_TRUE@ rm -rf check-runtime-dir runtime-testlog
@NATIVE_TRUE@ $(MKDIR_P) check-runtime-dir
@NATIVE_TRUE@ @abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
@ -777,7 +843,7 @@ mostlyclean-local:
@NATIVE_TRUE@ grep '^--- ' runtime-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
# check-cgo-test runs `go test misc/cgo/test` in our environment.
@NATIVE_TRUE@check-cgo-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
@NATIVE_TRUE@check-cgo-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
@NATIVE_TRUE@ rm -rf cgo-test-dir cgo-testlog
@NATIVE_TRUE@ $(MKDIR_P) cgo-test-dir/misc/cgo
@NATIVE_TRUE@ cp -r $(libgomiscdir)/cgo/test cgo-test-dir/misc/cgo/
@ -791,7 +857,7 @@ mostlyclean-local:
# check-carchive-test runs `go test misc/cgo/testcarchive/carchive_test.go`
# in our environment.
@NATIVE_TRUE@check-carchive-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
@NATIVE_TRUE@check-carchive-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
@NATIVE_TRUE@ rm -rf carchive-test-dir carchive-testlog
@NATIVE_TRUE@ $(MKDIR_P) carchive-test-dir/misc/cgo
@NATIVE_TRUE@ cp -r $(libgomiscdir)/cgo/testcarchive carchive-test-dir/misc/cgo/
@ -803,11 +869,25 @@ mostlyclean-local:
@NATIVE_TRUE@ (cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl}: go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog
@NATIVE_TRUE@ grep '^--- ' carchive-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
# check-vet runs `go test cmd/vet` in our environment.
@NATIVE_TRUE@check-vet: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
@NATIVE_TRUE@ rm -rf check-vet-dir cmd_vet-testlog
@NATIVE_TRUE@ $(MKDIR_P) check-vet-dir/src/cmd
@NATIVE_TRUE@ cp -r $(cmdsrcdir)/vet check-vet-dir/src/cmd/
@NATIVE_TRUE@ @abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
@NATIVE_TRUE@ abs_checkdir=`cd check-vet-dir && $(PWD_COMMAND)`; \
@NATIVE_TRUE@ echo "cd check-vet-dir/src/cmd/vet && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_vet-testlog
@NATIVE_TRUE@ $(CHECK_ENV) \
@NATIVE_TRUE@ GOPATH=`cd check-vet-dir && $(PWD_COMMAND)`; \
@NATIVE_TRUE@ export GOPATH; \
@NATIVE_TRUE@ (cd check-vet-dir/src/cmd/vet && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_vet-testlog 2>&1 || echo "--- $${fl}: go test cmd/vet (0.00s)" >> cmd_vet-testlog
@NATIVE_TRUE@ grep '^--- ' cmd_vet-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
# The check targets runs the tests and assembles the output files.
@NATIVE_TRUE@check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test
@NATIVE_TRUE@check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
@NATIVE_TRUE@ @mv gotools.head gotools.sum
@NATIVE_TRUE@ @cp gotools.sum gotools.log
@NATIVE_TRUE@ @for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog; do \
@NATIVE_TRUE@ @for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog cmd_vet-testlog; do \
@NATIVE_TRUE@ testname=`echo $${file} | sed -e 's/-testlog//' -e 's|_|/|'`; \
@NATIVE_TRUE@ echo "Running $${testname}" >> gotools.sum; \
@NATIVE_TRUE@ echo "Running $${testname}" >> gotools.log; \
@ -833,7 +913,7 @@ mostlyclean-local:
@NATIVE_TRUE@ @echo "runtest completed at `date`" >> gotools.log
@NATIVE_TRUE@ @if grep '^FAIL' gotools.sum >/dev/null 2>&1; then exit 1; fi
@NATIVE_TRUE@.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test
@NATIVE_TRUE@.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
# For a non-native build we have to build the programs using a
# previously built host (or build -> host) Go compiler. We should

2
libgo/MERGE
View file

@ -1,4 +1,4 @@
c8aec4095e089ff6ac50d18e97c3f46561f14f48
9ce6b5c2ed5d3d5251b9a6a0c548d5fb2c8567e8
The first line of this file holds the git revision number of the
last merge done from the master library sources.

59
libgo/Makefile.am
View file

@ -400,8 +400,11 @@ toolexeclibgounicode_DATA = \
# internal packages nothing will explicitly depend on them.
# Force them to be built.
noinst_DATA = \
golang_org/x/net/internal/nettest.gox \
golang_org/x/net/nettest.gox \
internal/testenv.gox \
net/internal/socktest.gox
net/internal/socktest.gox \
os/signal/internal/pty.gox
if LIBGO_IS_RTEMS
rtems_task_variable_add_file = runtime/rtems-task-variable-add.c
@ -533,6 +536,24 @@ s-version: Makefile
$(SHELL) $(srcdir)/mvifdiff.sh version.go.tmp version.go
$(STAMP) $@
objabi.go: s-objabi; @true
s-objabi: Makefile
rm -f objabi.go.tmp
echo "package objabi" > objabi.go.tmp
echo "import \"runtime\"" >> objabi.go.tmp
echo 'const defaultGOROOT = `$(prefix)`' >> objabi.go.tmp
echo 'const defaultGO386 = `sse2`' >> objabi.go.tmp
echo 'const defaultGOARM = `5`' >> objabi.go.tmp
echo 'const defaultGOMIPS = `hardfloat`' >> objabi.go.tmp
echo 'const defaultGOOS = runtime.GOOS' >> objabi.go.tmp
echo 'const defaultGOARCH = runtime.GOARCH' >> objabi.go.tmp
echo 'const defaultGO_EXTLINK_ENABLED = ``' >> objabi.go.tmp
echo 'const version = `'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'`' >> objabi.go.tmp
echo 'const stackGuardMultiplier = 1' >> objabi.go.tmp
echo 'const goexperiment = ``' >> objabi.go.tmp
$(SHELL) $(srcdir)/mvifdiff.sh objabi.go.tmp objabi.go
$(STAMP) $@
runtime_sysinfo.go: s-runtime_sysinfo; @true
s-runtime_sysinfo: $(srcdir)/mkrsysinfo.sh gen-sysinfo.go
GOARCH=$(GOARCH) GOOS=$(GOOS) $(SHELL) $(srcdir)/mkrsysinfo.sh
@ -553,10 +574,11 @@ zdefaultcc.go: s-zdefaultcc; @true
s-zdefaultcc: Makefile
echo 'package cfg' > zdefaultcc.go.tmp
echo >> zdefaultcc.go.tmp
echo 'const DefaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'const DefaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'const DefaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'func DefaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'func DefaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'func DefaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'const DefaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
echo 'var OSArchSupportsCgo = map[string]bool{}' >> zdefaultcc.go.tmp
$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
$(STAMP) $@
@ -758,11 +780,15 @@ PACKAGES = \
go/types \
golang_org/x/crypto/chacha20poly1305 \
golang_org/x/crypto/chacha20poly1305/internal/chacha20 \
golang_org/x/crypto/cryptobyte \
golang_org/x/crypto/cryptobyte/asn1 \
golang_org/x/crypto/curve25519 \
golang_org/x/crypto/poly1305 \
golang_org/x/net/http2/hpack \
golang_org/x/net/idna \
golang_org/x/net/internal/nettest \
golang_org/x/net/lex/httplex \
golang_org/x/net/nettest \
golang_org/x/net/proxy \
golang_org/x/text/secure/bidirule \
golang_org/x/text/transform \
@ -824,6 +850,7 @@ PACKAGES = \
os \
os/exec \
os/signal \
os/signal/internal/pty \
os/user \
path \
path/filepath \
@ -905,7 +932,7 @@ libgolibbegin_a_CFLAGS = $(AM_CFLAGS) -fPIC
GOTOOL_PACKAGES = \
cmd/go/internal/base \
cmd/go/internal/bug \
cmd/go/internal/buildid \
cmd/go/internal/cache \
cmd/go/internal/cfg \
cmd/go/internal/clean \
cmd/go/internal/cmdflag \
@ -927,7 +954,12 @@ GOTOOL_PACKAGES = \
cmd/go/internal/web \
cmd/go/internal/work \
cmd/internal/browser \
cmd/internal/objabi
cmd/internal/buildid \
cmd/internal/edit \
cmd/internal/objabi \
cmd/internal/test2json \
cmd/vet/internal/cfg \
cmd/vet/internal/whitelist
libgotool_a_SOURCES =
libgotool_a_DEPENDENCIES = $(addsuffix .lo,$(GOTOOL_PACKAGES))
@ -1136,17 +1168,23 @@ runtime_pprof_check_GOCFLAGS = -static-libgo -fno-inline
extra_go_files_runtime_internal_sys = version.go
runtime/internal/sys.lo.dep: $(extra_go_files_runtime_internal_sys)
extra_go_files_cmd_internal_objabi = objabi.go
cmd/internal/objabi.lo.dep: $(extra_go_files_cmd_internal_objabi)
extra_go_files_cmd_go_internal_cfg = zdefaultcc.go
cmd/go/internal/cfg.lo.dep: $(extra_go_files_cmd_go_internal_cfg)
extra_go_files_cmd_go_internal_load = zstdpkglist.go
cmd/go/internal/load.lo.dep: $(extra_go_files_cmd_go_internal_load)
extra_check_libs_cmd_go_internal_cache = $(abs_builddir)/libgotool.a
extra_check_libs_cmd_go_internal_generate = $(abs_builddir)/libgotool.a
extra_check_libs_cmd_go_internal_get = $(abs_builddir)/libgotool.a
extra_check_libs_cmd_go_internal_load = $(abs_builddir)/libgotool.a
extra_check_libs_cmd_go_internal_work = $(abs_builddir)/libgotool.a
extra_check_libs_cmd_vet_internal_cfg = $(abs_builddir)/libgotool.a
# FIXME: The following C files may as well move to the runtime
# directory and be treated like other C files.
@ -1233,10 +1271,12 @@ TEST_PACKAGES = \
bufio/check \
bytes/check \
context/check \
crypto/check \
errors/check \
expvar/check \
flag/check \
fmt/check \
hash/check \
html/check \
image/check \
io/check \
@ -1258,11 +1298,16 @@ TEST_PACKAGES = \
unicode/check \
archive/tar/check \
archive/zip/check \
cmd/go/internal/cache/check \
cmd/go/internal/generate/check \
cmd/go/internal/get/check \
cmd/go/internal/load/check \
cmd/go/internal/work/check \
cmd/internal/buildid/check \
cmd/internal/edit/check \
cmd/internal/objabi/check \
cmd/internal/test2json/check \
cmd/vet/internal/cfg/check \
compress/bzip2/check \
compress/flate/check \
compress/gzip/check \
@ -1315,6 +1360,7 @@ TEST_PACKAGES = \
go/constant/check \
go/doc/check \
go/format/check \
go/importer/check \
go/internal/gcimporter/check \
go/internal/gccgoimporter/check \
go/internal/srcimporter/check \
@ -1325,6 +1371,7 @@ TEST_PACKAGES = \
go/types/check \
golang_org/x/crypto/chacha20poly1305/check \
golang_org/x/crypto/chacha20poly1305/internal/chacha20/check \
golang_org/x/crypto/cryptobyte/check \
golang_org/x/crypto/curve25519/check \
golang_org/x/crypto/poly1305/check \
golang_org/x/net/http2/hpack/check \

56
libgo/Makefile.in
View file

@ -770,7 +770,9 @@ toolexeclibgounicode_DATA = \
# Some packages are only needed for tests, so unlike the other
# internal packages nothing will explicitly depend on them.
# Force them to be built.
noinst_DATA = internal/testenv.gox net/internal/socktest.gox \
noinst_DATA = golang_org/x/net/internal/nettest.gox \
golang_org/x/net/nettest.gox internal/testenv.gox \
net/internal/socktest.gox os/signal/internal/pty.gox \
zstdpkglist.go zdefaultcc.go
@LIBGO_IS_RTEMS_FALSE@rtems_task_variable_add_file =
@LIBGO_IS_RTEMS_TRUE@rtems_task_variable_add_file = runtime/rtems-task-variable-add.c
@ -909,11 +911,15 @@ PACKAGES = \
go/types \
golang_org/x/crypto/chacha20poly1305 \
golang_org/x/crypto/chacha20poly1305/internal/chacha20 \
golang_org/x/crypto/cryptobyte \
golang_org/x/crypto/cryptobyte/asn1 \
golang_org/x/crypto/curve25519 \
golang_org/x/crypto/poly1305 \
golang_org/x/net/http2/hpack \
golang_org/x/net/idna \
golang_org/x/net/internal/nettest \
golang_org/x/net/lex/httplex \
golang_org/x/net/nettest \
golang_org/x/net/proxy \
golang_org/x/text/secure/bidirule \
golang_org/x/text/transform \
@ -975,6 +981,7 @@ PACKAGES = \
os \
os/exec \
os/signal \
os/signal/internal/pty \
os/user \
path \
path/filepath \
@ -1053,7 +1060,7 @@ libgolibbegin_a_CFLAGS = $(AM_CFLAGS) -fPIC
GOTOOL_PACKAGES = \
cmd/go/internal/base \
cmd/go/internal/bug \
cmd/go/internal/buildid \
cmd/go/internal/cache \
cmd/go/internal/cfg \
cmd/go/internal/clean \
cmd/go/internal/cmdflag \
@ -1075,7 +1082,12 @@ GOTOOL_PACKAGES = \
cmd/go/internal/web \
cmd/go/internal/work \
cmd/internal/browser \
cmd/internal/objabi
cmd/internal/buildid \
cmd/internal/edit \
cmd/internal/objabi \
cmd/internal/test2json \
cmd/vet/internal/cfg \
cmd/vet/internal/whitelist
libgotool_a_SOURCES =
libgotool_a_DEPENDENCIES = $(addsuffix .lo,$(GOTOOL_PACKAGES))
@ -1210,12 +1222,15 @@ runtime_internal_sys_lo_check_GOCFLAGS = -fgo-compiling-runtime
# Also use -fno-inline to get better results from the memory profiler.
runtime_pprof_check_GOCFLAGS = -static-libgo -fno-inline
extra_go_files_runtime_internal_sys = version.go
extra_go_files_cmd_internal_objabi = objabi.go
extra_go_files_cmd_go_internal_cfg = zdefaultcc.go
extra_go_files_cmd_go_internal_load = zstdpkglist.go
extra_check_libs_cmd_go_internal_cache = $(abs_builddir)/libgotool.a
extra_check_libs_cmd_go_internal_generate = $(abs_builddir)/libgotool.a
extra_check_libs_cmd_go_internal_get = $(abs_builddir)/libgotool.a
extra_check_libs_cmd_go_internal_load = $(abs_builddir)/libgotool.a
extra_check_libs_cmd_go_internal_work = $(abs_builddir)/libgotool.a
extra_check_libs_cmd_vet_internal_cfg = $(abs_builddir)/libgotool.a
@HAVE_STAT_TIMESPEC_FALSE@@LIBGO_IS_SOLARIS_TRUE@matchargs_os =
# Solaris 11.4 changed the type of fields in struct stat.
@ -1238,10 +1253,12 @@ TEST_PACKAGES = \
bufio/check \
bytes/check \
context/check \
crypto/check \
errors/check \
expvar/check \
flag/check \
fmt/check \
hash/check \
html/check \
image/check \
io/check \
@ -1263,11 +1280,16 @@ TEST_PACKAGES = \
unicode/check \
archive/tar/check \
archive/zip/check \
cmd/go/internal/cache/check \
cmd/go/internal/generate/check \
cmd/go/internal/get/check \
cmd/go/internal/load/check \
cmd/go/internal/work/check \
cmd/internal/buildid/check \
cmd/internal/edit/check \
cmd/internal/objabi/check \
cmd/internal/test2json/check \
cmd/vet/internal/cfg/check \
compress/bzip2/check \
compress/flate/check \
compress/gzip/check \
@ -1320,6 +1342,7 @@ TEST_PACKAGES = \
go/constant/check \
go/doc/check \
go/format/check \
go/importer/check \
go/internal/gcimporter/check \
go/internal/gccgoimporter/check \
go/internal/srcimporter/check \
@ -1330,6 +1353,7 @@ TEST_PACKAGES = \
go/types/check \
golang_org/x/crypto/chacha20poly1305/check \
golang_org/x/crypto/chacha20poly1305/internal/chacha20/check \
golang_org/x/crypto/cryptobyte/check \
golang_org/x/crypto/curve25519/check \
golang_org/x/crypto/poly1305/check \
golang_org/x/net/http2/hpack/check \
@ -3130,6 +3154,24 @@ s-version: Makefile
$(SHELL) $(srcdir)/mvifdiff.sh version.go.tmp version.go
$(STAMP) $@
objabi.go: s-objabi; @true
s-objabi: Makefile
rm -f objabi.go.tmp
echo "package objabi" > objabi.go.tmp
echo "import \"runtime\"" >> objabi.go.tmp
echo 'const defaultGOROOT = `$(prefix)`' >> objabi.go.tmp
echo 'const defaultGO386 = `sse2`' >> objabi.go.tmp
echo 'const defaultGOARM = `5`' >> objabi.go.tmp
echo 'const defaultGOMIPS = `hardfloat`' >> objabi.go.tmp
echo 'const defaultGOOS = runtime.GOOS' >> objabi.go.tmp
echo 'const defaultGOARCH = runtime.GOARCH' >> objabi.go.tmp
echo 'const defaultGO_EXTLINK_ENABLED = ``' >> objabi.go.tmp
echo 'const version = `'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'`' >> objabi.go.tmp
echo 'const stackGuardMultiplier = 1' >> objabi.go.tmp
echo 'const goexperiment = ``' >> objabi.go.tmp
$(SHELL) $(srcdir)/mvifdiff.sh objabi.go.tmp objabi.go
$(STAMP) $@
runtime_sysinfo.go: s-runtime_sysinfo; @true
s-runtime_sysinfo: $(srcdir)/mkrsysinfo.sh gen-sysinfo.go
GOARCH=$(GOARCH) GOOS=$(GOOS) $(SHELL) $(srcdir)/mkrsysinfo.sh
@ -3146,10 +3188,11 @@ zdefaultcc.go: s-zdefaultcc; @true
s-zdefaultcc: Makefile
echo 'package cfg' > zdefaultcc.go.tmp
echo >> zdefaultcc.go.tmp
echo 'const DefaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'const DefaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'const DefaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
echo 'func DefaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'func DefaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'func DefaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
echo 'const DefaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
echo 'var OSArchSupportsCgo = map[string]bool{}' >> zdefaultcc.go.tmp
$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
$(STAMP) $@
@ -3305,6 +3348,7 @@ $(foreach package,$(GOTOOL_PACKAGES),$(eval $(call PACKAGE_template,$(package)))
runtime.lo.dep: $(extra_go_files_runtime)
syscall.lo.dep: $(extra_go_files_syscall)
runtime/internal/sys.lo.dep: $(extra_go_files_runtime_internal_sys)
cmd/internal/objabi.lo.dep: $(extra_go_files_cmd_internal_objabi)
cmd/go/internal/cfg.lo.dep: $(extra_go_files_cmd_go_internal_cfg)
cmd/go/internal/load.lo.dep: $(extra_go_files_cmd_go_internal_load)

2
libgo/VERSION
View file

@ -1 +1 @@
go1.9
go1.10beta1

24
libgo/configure vendored
View file

@ -2494,7 +2494,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
ac_config_headers="$ac_config_headers config.h"
libtool_VERSION=12:0:0
libtool_VERSION=13:0:0
# Default to --enable-multilib
@ -13652,7 +13652,7 @@ ALLGOARCHFAMILY="I386 ALPHA AMD64 ARM ARM64 IA64 M68K MIPS MIPS64 PPC PPC64 S390
GOARCH=unknown
GOARCH_FAMILY=unknown
GOARCH_BIGENDIAN=0
GOARCH_BIGENDIAN=false
GOARCH_CACHELINESIZE=64
GOARCH_PHYSPAGESIZE=4096
GOARCH_PCQUANTUM=1
@ -13680,6 +13680,12 @@ case ${host} in
GOARCH_CACHELINESIZE=32
GOARCH_PCQUANTUM=4
GOARCH_MINFRAMESIZE=4
case ${host} in
arm*b*-*-*)
GOARCH=armbe
GOARCH_BIGENDIAN=true
;;
esac
;;
i[34567]86-*-* | x86_64-*-*)
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
@ -13712,7 +13718,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
m68k*-*-*)
GOARCH=m68k
GOARCH_FAMILY=M68K
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
GOARCH_CACHELINESIZE=16
GOARCH_PCQUANTUM=4
GOARCH_INT64ALIGN=2
@ -13776,7 +13782,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
GOARCH="${GOARCH}le"
;;
*)
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
;;
esac
GOARCH_CACHELINESIZE=32
@ -13794,7 +13800,7 @@ _ACEOF
if ac_fn_c_try_compile "$LINENO"; then :
GOARCH=ppc
GOARCH_FAMILY=PPC
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
else
@ -13811,7 +13817,7 @@ if ac_fn_c_try_compile "$LINENO"; then :
else
GOARCH=ppc64
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
fi
rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
@ -13841,7 +13847,7 @@ GOARCH_MINFRAMESIZE=8
fi
rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
GOARCH_CACHELINESIZE=256
GOARCH_PCQUANTUM=2
;;
@ -13863,7 +13869,7 @@ GOARCH_FAMILY=SPARC64
fi
rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
GOARCH_PHYSPAGESIZE=8192
GOARCH_PCQUANTUM=4
;;
@ -15142,7 +15148,7 @@ fi
$as_echo "$libgo_cv_c_fancymath" >&6; }
MATH_FLAG=
if test "$libgo_cv_c_fancymath" = yes; then
MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations"
MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations -fno-math-errno"
else
MATH_FLAG="-ffp-contract=off"
fi

24
libgo/configure.ac
View file

@ -11,7 +11,7 @@ AC_INIT(package-unused, version-unused,, libgo)
AC_CONFIG_SRCDIR(Makefile.am)
AC_CONFIG_HEADER(config.h)
libtool_VERSION=12:0:0
libtool_VERSION=13:0:0
AC_SUBST(libtool_VERSION)
AM_ENABLE_MULTILIB(, ..)
@ -215,7 +215,7 @@ ALLGOARCHFAMILY="I386 ALPHA AMD64 ARM ARM64 IA64 M68K MIPS MIPS64 PPC PPC64 S390
GOARCH=unknown
GOARCH_FAMILY=unknown
GOARCH_BIGENDIAN=0
GOARCH_BIGENDIAN=false
GOARCH_CACHELINESIZE=64
GOARCH_PHYSPAGESIZE=4096
GOARCH_PCQUANTUM=1
@ -243,6 +243,12 @@ case ${host} in
GOARCH_CACHELINESIZE=32
GOARCH_PCQUANTUM=4
GOARCH_MINFRAMESIZE=4
case ${host} in
arm*b*-*-*)
GOARCH=armbe
GOARCH_BIGENDIAN=true
;;
esac
;;
changequote(,)dnl
i[34567]86-*-* | x86_64-*-*)
@ -270,7 +276,7 @@ GOARCH_HUGEPAGESIZE="1 << 21"
m68k*-*-*)
GOARCH=m68k
GOARCH_FAMILY=M68K
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
GOARCH_CACHELINESIZE=16
GOARCH_PCQUANTUM=4
GOARCH_INT64ALIGN=2
@ -313,7 +319,7 @@ GOARCH_HUGEPAGESIZE="1 << 21"
GOARCH="${GOARCH}le"
;;
*)
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
;;
esac
GOARCH_CACHELINESIZE=32
@ -327,7 +333,7 @@ GOARCH_HUGEPAGESIZE="1 << 21"
#endif],
[GOARCH=ppc
GOARCH_FAMILY=PPC
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
],
[
GOARCH_FAMILY=PPC64
@ -338,7 +344,7 @@ AC_COMPILE_IFELSE([
[GOARCH=ppc64le
],
[GOARCH=ppc64
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
])])
GOARCH_PHYSPAGESIZE=65536
GOARCH_PCQUANTUM=4
@ -356,7 +362,7 @@ GOARCH_MINFRAMESIZE=4
GOARCH_FAMILY=S390X
GOARCH_MINFRAMESIZE=8
])
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
GOARCH_CACHELINESIZE=256
GOARCH_PCQUANTUM=2
;;
@ -371,7 +377,7 @@ GOARCH_FAMILY=SPARC
[GOARCH=sparc64
GOARCH_FAMILY=SPARC64
])
GOARCH_BIGENDIAN=1
GOARCH_BIGENDIAN=true
GOARCH_PHYSPAGESIZE=8192
GOARCH_PCQUANTUM=4
;;
@ -718,7 +724,7 @@ AC_COMPILE_IFELSE([int i;],
CFLAGS=$CFLAGS_hold])
MATH_FLAG=
if test "$libgo_cv_c_fancymath" = yes; then
MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations"
MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations -fno-math-errno"
else
MATH_FLAG="-ffp-contract=off"
fi

610
libgo/go/archive/tar/common.go
View file

@ -3,20 +3,22 @@
// license that can be found in the LICENSE file.
// Package tar implements access to tar archives.
// It aims to cover most of the variations, including those produced
// by GNU and BSD tars.
//
// References:
// http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5
// http://www.gnu.org/software/tar/manual/html_node/Standard.html
// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html
// Tape archives (tar) are a file format for storing a sequence of files that
// can be read and written in a streaming manner.
// This package aims to cover most variations of the format,
// including those produced by GNU and BSD tar tools.
package tar
import (
"errors"
"fmt"
"math"
"os"
"path"
"reflect"
"strconv"
"strings"
"time"
)
@ -24,42 +26,500 @@ import (
// architectures. If a large value is encountered when decoding, the result
// stored in Header will be the truncated version.
// Header type flags.
const (
TypeReg = '0' // regular file
TypeRegA = '\x00' // regular file
TypeLink = '1' // hard link
TypeSymlink = '2' // symbolic link
TypeChar = '3' // character device node
TypeBlock = '4' // block device node
TypeDir = '5' // directory
TypeFifo = '6' // fifo node
TypeCont = '7' // reserved
TypeXHeader = 'x' // extended header
TypeXGlobalHeader = 'g' // global extended header
TypeGNULongName = 'L' // Next file has a long name
TypeGNULongLink = 'K' // Next file symlinks to a file w/ a long name
TypeGNUSparse = 'S' // sparse file
var (
ErrHeader = errors.New("archive/tar: invalid tar header")
ErrWriteTooLong = errors.New("archive/tar: write too long")
ErrFieldTooLong = errors.New("archive/tar: header field too long")
ErrWriteAfterClose = errors.New("archive/tar: write after close")
errMissData = errors.New("archive/tar: sparse file references non-existent data")
errUnrefData = errors.New("archive/tar: sparse file contains unreferenced data")
errWriteHole = errors.New("archive/tar: write non-NUL byte in sparse hole")
)
type headerError []string
func (he headerError) Error() string {
const prefix = "archive/tar: cannot encode header"
var ss []string
for _, s := range he {
if s != "" {
ss = append(ss, s)
}
}
if len(ss) == 0 {
return prefix
}
return fmt.Sprintf("%s: %v", prefix, strings.Join(ss, "; and "))
}
// Type flags for Header.Typeflag.
const (
// Type '0' indicates a regular file.
TypeReg = '0'
TypeRegA = '\x00' // For legacy support; use TypeReg instead
// Type '1' to '6' are header-only flags and may not have a data body.
TypeLink = '1' // Hard link
TypeSymlink = '2' // Symbolic link
TypeChar = '3' // Character device node
TypeBlock = '4' // Block device node
TypeDir = '5' // Directory
TypeFifo = '6' // FIFO node
// Type '7' is reserved.
TypeCont = '7'
// Type 'x' is used by the PAX format to store key-value records that
// are only relevant to the next file.
// This package transparently handles these types.
TypeXHeader = 'x'
// Type 'g' is used by the PAX format to store key-value records that
// are relevant to all subsequent files.
// This package only supports parsing and composing such headers,
// but does not currently support persisting the global state across files.
TypeXGlobalHeader = 'g'
// Type 'S' indicates a sparse file in the GNU format.
TypeGNUSparse = 'S'
// Types 'L' and 'K' are used by the GNU format for a meta file
// used to store the path or link name for the next file.
// This package transparently handles these types.
TypeGNULongName = 'L'
TypeGNULongLink = 'K'
)
// Keywords for PAX extended header records.
const (
paxNone = "" // Indicates that no PAX key is suitable
paxPath = "path"
paxLinkpath = "linkpath"
paxSize = "size"
paxUid = "uid"
paxGid = "gid"
paxUname = "uname"
paxGname = "gname"
paxMtime = "mtime"
paxAtime = "atime"
paxCtime = "ctime" // Removed from later revision of PAX spec, but was valid
paxCharset = "charset" // Currently unused
paxComment = "comment" // Currently unused
paxSchilyXattr = "SCHILY.xattr."
// Keywords for GNU sparse files in a PAX extended header.
paxGNUSparse = "GNU.sparse."
paxGNUSparseNumBlocks = "GNU.sparse.numblocks"
paxGNUSparseOffset = "GNU.sparse.offset"
paxGNUSparseNumBytes = "GNU.sparse.numbytes"
paxGNUSparseMap = "GNU.sparse.map"
paxGNUSparseName = "GNU.sparse.name"
paxGNUSparseMajor = "GNU.sparse.major"
paxGNUSparseMinor = "GNU.sparse.minor"
paxGNUSparseSize = "GNU.sparse.size"
paxGNUSparseRealSize = "GNU.sparse.realsize"
)
// basicKeys is a set of the PAX keys for which we have built-in support.
// This does not contain "charset" or "comment", which are both PAX-specific,
// so adding them as first-class features of Header is unlikely.
// Users can use the PAXRecords field to set it themselves.
var basicKeys = map[string]bool{
paxPath: true, paxLinkpath: true, paxSize: true, paxUid: true, paxGid: true,
paxUname: true, paxGname: true, paxMtime: true, paxAtime: true, paxCtime: true,
}
// A Header represents a single header in a tar archive.
// Some fields may not be populated.
//
// For forward compatibility, users that retrieve a Header from Reader.Next,
// mutate it in some ways, and then pass it back to Writer.WriteHeader
// should do so by creating a new Header and copying the fields
// that they are interested in preserving.
type Header struct {
Name string // name of header file entry
Mode int64 // permission and mode bits
Uid int // user id of owner
Gid int // group id of owner
Size int64 // length in bytes
ModTime time.Time // modified time
Typeflag byte // type of header entry
Linkname string // target name of link
Uname string // user name of owner
Gname string // group name of owner
Devmajor int64 // major number of character or block device
Devminor int64 // minor number of character or block device
AccessTime time.Time // access time
ChangeTime time.Time // status change time
Xattrs map[string]string
Typeflag byte // Type of header entry (should be TypeReg for most files)
Name string // Name of file entry
Linkname string // Target name of link (valid for TypeLink or TypeSymlink)
Size int64 // Logical file size in bytes
Mode int64 // Permission and mode bits
Uid int // User ID of owner
Gid int // Group ID of owner
Uname string // User name of owner
Gname string // Group name of owner
// If the Format is unspecified, then Writer.WriteHeader rounds ModTime
// to the nearest second and ignores the AccessTime and ChangeTime fields.
//
// To use AccessTime or ChangeTime, specify the Format as PAX or GNU.
// To use sub-second resolution, specify the Format as PAX.
ModTime time.Time // Modification time
AccessTime time.Time // Access time (requires either PAX or GNU support)
ChangeTime time.Time // Change time (requires either PAX or GNU support)
Devmajor int64 // Major device number (valid for TypeChar or TypeBlock)
Devminor int64 // Minor device number (valid for TypeChar or TypeBlock)
// Xattrs stores extended attributes as PAX records under the
// "SCHILY.xattr." namespace.
//
// The following are semantically equivalent:
// h.Xattrs[key] = value
// h.PAXRecords["SCHILY.xattr."+key] = value
//
// When Writer.WriteHeader is called, the contents of Xattrs will take
// precedence over those in PAXRecords.
//
// Deprecated: Use PAXRecords instead.
Xattrs map[string]string
// PAXRecords is a map of PAX extended header records.
//
// User-defined records should have keys of the following form:
// VENDOR.keyword
// Where VENDOR is some namespace in all uppercase, and keyword may
// not contain the '=' character (e.g., "GOLANG.pkg.version").
// The key and value should be non-empty UTF-8 strings.
//
// When Writer.WriteHeader is called, PAX records derived from the
// the other fields in Header take precedence over PAXRecords.
PAXRecords map[string]string
// Format specifies the format of the tar header.
//
// This is set by Reader.Next as a best-effort guess at the format.
// Since the Reader liberally reads some non-compliant files,
// it is possible for this to be FormatUnknown.
//
// If the format is unspecified when Writer.WriteHeader is called,
// then it uses the first format (in the order of USTAR, PAX, GNU)
// capable of encoding this Header (see Format).
Format Format
}
// sparseEntry represents a Length-sized fragment at Offset in the file.
type sparseEntry struct{ Offset, Length int64 }
func (s sparseEntry) endOffset() int64 { return s.Offset + s.Length }
// A sparse file can be represented as either a sparseDatas or a sparseHoles.
// As long as the total size is known, they are equivalent and one can be
// converted to the other form and back. The various tar formats with sparse
// file support represent sparse files in the sparseDatas form. That is, they
// specify the fragments in the file that has data, and treat everything else as
// having zero bytes. As such, the encoding and decoding logic in this package
// deals with sparseDatas.
//
// However, the external API uses sparseHoles instead of sparseDatas because the
// zero value of sparseHoles logically represents a normal file (i.e., there are
// no holes in it). On the other hand, the zero value of sparseDatas implies
// that the file has no data in it, which is rather odd.
//
// As an example, if the underlying raw file contains the 10-byte data:
// var compactFile = "abcdefgh"
//
// And the sparse map has the following entries:
// var spd sparseDatas = []sparseEntry{
// {Offset: 2, Length: 5}, // Data fragment for 2..6
// {Offset: 18, Length: 3}, // Data fragment for 18..20
// }
// var sph sparseHoles = []sparseEntry{
// {Offset: 0, Length: 2}, // Hole fragment for 0..1
// {Offset: 7, Length: 11}, // Hole fragment for 7..17
// {Offset: 21, Length: 4}, // Hole fragment for 21..24
// }
//
// Then the content of the resulting sparse file with a Header.Size of 25 is:
// var sparseFile = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4
type (
sparseDatas []sparseEntry
sparseHoles []sparseEntry
)
// validateSparseEntries reports whether sp is a valid sparse map.
// It does not matter whether sp represents data fragments or hole fragments.
func validateSparseEntries(sp []sparseEntry, size int64) bool {
// Validate all sparse entries. These are the same checks as performed by
// the BSD tar utility.
if size < 0 {
return false
}
var pre sparseEntry
for _, cur := range sp {
switch {
case cur.Offset < 0 || cur.Length < 0:
return false // Negative values are never okay
case cur.Offset > math.MaxInt64-cur.Length:
return false // Integer overflow with large length
case cur.endOffset() > size:
return false // Region extends beyond the actual size
case pre.endOffset() > cur.Offset:
return false // Regions cannot overlap and must be in order
}
pre = cur
}
return true
}
// alignSparseEntries mutates src and returns dst where each fragment's
// starting offset is aligned up to the nearest block edge, and each
// ending offset is aligned down to the nearest block edge.
//
// Even though the Go tar Reader and the BSD tar utility can handle entries
// with arbitrary offsets and lengths, the GNU tar utility can only handle
// offsets and lengths that are multiples of blockSize.
func alignSparseEntries(src []sparseEntry, size int64) []sparseEntry {
dst := src[:0]
for _, s := range src {
pos, end := s.Offset, s.endOffset()
pos += blockPadding(+pos) // Round-up to nearest blockSize
if end != size {
end -= blockPadding(-end) // Round-down to nearest blockSize
}
if pos < end {
dst = append(dst, sparseEntry{Offset: pos, Length: end - pos})
}
}
return dst
}
// invertSparseEntries converts a sparse map from one form to the other.
// If the input is sparseHoles, then it will output sparseDatas and vice-versa.
// The input must have been already validated.
//
// This function mutates src and returns a normalized map where:
// * adjacent fragments are coalesced together
// * only the last fragment may be empty
// * the endOffset of the last fragment is the total size
func invertSparseEntries(src []sparseEntry, size int64) []sparseEntry {
dst := src[:0]
var pre sparseEntry
for _, cur := range src {
if cur.Length == 0 {
continue // Skip empty fragments
}
pre.Length = cur.Offset - pre.Offset
if pre.Length > 0 {
dst = append(dst, pre) // Only add non-empty fragments
}
pre.Offset = cur.endOffset()
}
pre.Length = size - pre.Offset // Possibly the only empty fragment
return append(dst, pre)
}
// fileState tracks the number of logical (includes sparse holes) and physical
// (actual in tar archive) bytes remaining for the current file.
//
// Invariant: LogicalRemaining >= PhysicalRemaining
type fileState interface {
LogicalRemaining() int64
PhysicalRemaining() int64
}
// allowedFormats determines which formats can be used.
// The value returned is the logical OR of multiple possible formats.
// If the value is FormatUnknown, then the input Header cannot be encoded
// and an error is returned explaining why.
//
// As a by-product of checking the fields, this function returns paxHdrs, which
// contain all fields that could not be directly encoded.
// A value receiver ensures that this method does not mutate the source Header.
func (h Header) allowedFormats() (format Format, paxHdrs map[string]string, err error) {
format = FormatUSTAR | FormatPAX | FormatGNU
paxHdrs = make(map[string]string)
var whyNoUSTAR, whyNoPAX, whyNoGNU string
var preferPAX bool // Prefer PAX over USTAR
verifyString := func(s string, size int, name, paxKey string) {
// NUL-terminator is optional for path and linkpath.
// Technically, it is required for uname and gname,
// but neither GNU nor BSD tar checks for it.
tooLong := len(s) > size
allowLongGNU := paxKey == paxPath || paxKey == paxLinkpath
if hasNUL(s) || (tooLong && !allowLongGNU) {
whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%q", name, s)
format.mustNotBe(FormatGNU)
}
if !isASCII(s) || tooLong {
canSplitUSTAR := paxKey == paxPath
if _, _, ok := splitUSTARPath(s); !canSplitUSTAR || !ok {
whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%q", name, s)
format.mustNotBe(FormatUSTAR)
}
if paxKey == paxNone {
whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%q", name, s)
format.mustNotBe(FormatPAX)
} else {
paxHdrs[paxKey] = s
}
}
if v, ok := h.PAXRecords[paxKey]; ok && v == s {
paxHdrs[paxKey] = v
}
}
verifyNumeric := func(n int64, size int, name, paxKey string) {
if !fitsInBase256(size, n) {
whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%d", name, n)
format.mustNotBe(FormatGNU)
}
if !fitsInOctal(size, n) {
whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%d", name, n)
format.mustNotBe(FormatUSTAR)
if paxKey == paxNone {
whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%d", name, n)
format.mustNotBe(FormatPAX)
} else {
paxHdrs[paxKey] = strconv.FormatInt(n, 10)
}
}
if v, ok := h.PAXRecords[paxKey]; ok && v == strconv.FormatInt(n, 10) {
paxHdrs[paxKey] = v
}
}
verifyTime := func(ts time.Time, size int, name, paxKey string) {
if ts.IsZero() {
return // Always okay
}
if !fitsInBase256(size, ts.Unix()) {
whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%v", name, ts)
format.mustNotBe(FormatGNU)
}
isMtime := paxKey == paxMtime
fitsOctal := fitsInOctal(size, ts.Unix())
if (isMtime && !fitsOctal) || !isMtime {
whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%v", name, ts)
format.mustNotBe(FormatUSTAR)
}
needsNano := ts.Nanosecond() != 0
if !isMtime || !fitsOctal || needsNano {
preferPAX = true // USTAR may truncate sub-second measurements
if paxKey == paxNone {
whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%v", name, ts)
format.mustNotBe(FormatPAX)
} else {
paxHdrs[paxKey] = formatPAXTime(ts)
}
}
if v, ok := h.PAXRecords[paxKey]; ok && v == formatPAXTime(ts) {
paxHdrs[paxKey] = v
}
}
// Check basic fields.
var blk block
v7 := blk.V7()
ustar := blk.USTAR()
gnu := blk.GNU()
verifyString(h.Name, len(v7.Name()), "Name", paxPath)
verifyString(h.Linkname, len(v7.LinkName()), "Linkname", paxLinkpath)
verifyString(h.Uname, len(ustar.UserName()), "Uname", paxUname)
verifyString(h.Gname, len(ustar.GroupName()), "Gname", paxGname)
verifyNumeric(h.Mode, len(v7.Mode()), "Mode", paxNone)
verifyNumeric(int64(h.Uid), len(v7.UID()), "Uid", paxUid)
verifyNumeric(int64(h.Gid), len(v7.GID()), "Gid", paxGid)
verifyNumeric(h.Size, len(v7.Size()), "Size", paxSize)
verifyNumeric(h.Devmajor, len(ustar.DevMajor()), "Devmajor", paxNone)
verifyNumeric(h.Devminor, len(ustar.DevMinor()), "Devminor", paxNone)
verifyTime(h.ModTime, len(v7.ModTime()), "ModTime", paxMtime)
verifyTime(h.AccessTime, len(gnu.AccessTime()), "AccessTime", paxAtime)
verifyTime(h.ChangeTime, len(gnu.ChangeTime()), "ChangeTime", paxCtime)
// Check for header-only types.
var whyOnlyPAX, whyOnlyGNU string
switch h.Typeflag {
case TypeReg, TypeChar, TypeBlock, TypeFifo, TypeGNUSparse:
// Exclude TypeLink and TypeSymlink, since they may reference directories.
if strings.HasSuffix(h.Name, "/") {
return FormatUnknown, nil, headerError{"filename may not have trailing slash"}
}
case TypeXHeader, TypeGNULongName, TypeGNULongLink:
return FormatUnknown, nil, headerError{"cannot manually encode TypeXHeader, TypeGNULongName, or TypeGNULongLink headers"}
case TypeXGlobalHeader:
h2 := Header{Name: h.Name, Typeflag: h.Typeflag, Xattrs: h.Xattrs, PAXRecords: h.PAXRecords, Format: h.Format}
if !reflect.DeepEqual(h, h2) {
return FormatUnknown, nil, headerError{"only PAXRecords should be set for TypeXGlobalHeader"}
}
whyOnlyPAX = "only PAX supports TypeXGlobalHeader"
format.mayOnlyBe(FormatPAX)
}
if !isHeaderOnlyType(h.Typeflag) && h.Size < 0 {
return FormatUnknown, nil, headerError{"negative size on header-only type"}
}
// Check PAX records.
if len(h.Xattrs) > 0 {
for k, v := range h.Xattrs {
paxHdrs[paxSchilyXattr+k] = v
}
whyOnlyPAX = "only PAX supports Xattrs"
format.mayOnlyBe(FormatPAX)
}
if len(h.PAXRecords) > 0 {
for k, v := range h.PAXRecords {
switch _, exists := paxHdrs[k]; {
case exists:
continue // Do not overwrite existing records
case h.Typeflag == TypeXGlobalHeader:
paxHdrs[k] = v // Copy all records
case !basicKeys[k] && !strings.HasPrefix(k, paxGNUSparse):
paxHdrs[k] = v // Ignore local records that may conflict
}
}
whyOnlyPAX = "only PAX supports PAXRecords"
format.mayOnlyBe(FormatPAX)
}
for k, v := range paxHdrs {
if !validPAXRecord(k, v) {
return FormatUnknown, nil, headerError{fmt.Sprintf("invalid PAX record: %q", k+" = "+v)}
}
}
// TODO(dsnet): Re-enable this when adding sparse support.
// See https://golang.org/issue/22735
/*
// Check sparse files.
if len(h.SparseHoles) > 0 || h.Typeflag == TypeGNUSparse {
if isHeaderOnlyType(h.Typeflag) {
return FormatUnknown, nil, headerError{"header-only type cannot be sparse"}
}
if !validateSparseEntries(h.SparseHoles, h.Size) {
return FormatUnknown, nil, headerError{"invalid sparse holes"}
}
if h.Typeflag == TypeGNUSparse {
whyOnlyGNU = "only GNU supports TypeGNUSparse"
format.mayOnlyBe(FormatGNU)
} else {
whyNoGNU = "GNU supports sparse files only with TypeGNUSparse"
format.mustNotBe(FormatGNU)
}
whyNoUSTAR = "USTAR does not support sparse files"
format.mustNotBe(FormatUSTAR)
}
*/
// Check desired format.
if wantFormat := h.Format; wantFormat != FormatUnknown {
if wantFormat.has(FormatPAX) && !preferPAX {
wantFormat.mayBe(FormatUSTAR) // PAX implies USTAR allowed too
}
format.mayOnlyBe(wantFormat) // Set union of formats allowed and format wanted
}
if format == FormatUnknown {
switch h.Format {
case FormatUSTAR:
err = headerError{"Format specifies USTAR", whyNoUSTAR, whyOnlyPAX, whyOnlyGNU}
case FormatPAX:
err = headerError{"Format specifies PAX", whyNoPAX, whyOnlyGNU}
case FormatGNU:
err = headerError{"Format specifies GNU", whyNoGNU, whyOnlyPAX}
default:
err = headerError{whyNoUSTAR, whyNoPAX, whyNoGNU, whyOnlyPAX, whyOnlyGNU}
}
}
return format, paxHdrs, err
}
// FileInfo returns an os.FileInfo for the Header.
@ -92,63 +552,43 @@ func (fi headerFileInfo) Mode() (mode os.FileMode) {
// Set setuid, setgid and sticky bits.
if fi.h.Mode&c_ISUID != 0 {
// setuid
mode |= os.ModeSetuid
}
if fi.h.Mode&c_ISGID != 0 {
// setgid
mode |= os.ModeSetgid
}
if fi.h.Mode&c_ISVTX != 0 {
// sticky
mode |= os.ModeSticky
}
// Set file mode bits.
// clear perm, setuid, setgid and sticky bits.
m := os.FileMode(fi.h.Mode) &^ 07777
if m == c_ISDIR {
// directory
// Set file mode bits; clear perm, setuid, setgid, and sticky bits.
switch m := os.FileMode(fi.h.Mode) &^ 07777; m {
case c_ISDIR:
mode |= os.ModeDir
}
if m == c_ISFIFO {
// named pipe (FIFO)
case c_ISFIFO:
mode |= os.ModeNamedPipe
}
if m == c_ISLNK {
// symbolic link
case c_ISLNK:
mode |= os.ModeSymlink
}
if m == c_ISBLK {
// device file
case c_ISBLK:
mode |= os.ModeDevice
}
if m == c_ISCHR {
// Unix character device
case c_ISCHR:
mode |= os.ModeDevice
mode |= os.ModeCharDevice
}
if m == c_ISSOCK {
// Unix domain socket
case c_ISSOCK:
mode |= os.ModeSocket
}
switch fi.h.Typeflag {
case TypeSymlink:
// symbolic link
mode |= os.ModeSymlink
case TypeChar:
// character device node
mode |= os.ModeDevice
mode |= os.ModeCharDevice
case TypeBlock:
// block device node
mode |= os.ModeDevice
case TypeDir:
// directory
mode |= os.ModeDir
case TypeFifo:
// fifo node
mode |= os.ModeNamedPipe
}
@ -176,33 +616,16 @@ const (
c_ISSOCK = 0140000 // Socket
)
// Keywords for the PAX Extended Header
const (
paxAtime = "atime"
paxCharset = "charset"
paxComment = "comment"
paxCtime = "ctime" // please note that ctime is not a valid pax header.
paxGid = "gid"
paxGname = "gname"
paxLinkpath = "linkpath"
paxMtime = "mtime"
paxPath = "path"
paxSize = "size"
paxUid = "uid"
paxUname = "uname"
paxXattr = "SCHILY.xattr."
paxNone = ""
)
// FileInfoHeader creates a partially-populated Header from fi.
// If fi describes a symlink, FileInfoHeader records link as the link target.
// If fi describes a directory, a slash is appended to the name.
// Because os.FileInfo's Name method returns only the base name of
// the file it describes, it may be necessary to modify the Name field
// of the returned header to provide the full path name of the file.
//
// Since os.FileInfo's Name method only returns the base name of
// the file it describes, it may be necessary to modify Header.Name
// to provide the full path name of the file.
func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
if fi == nil {
return nil, errors.New("tar: FileInfo is nil")
return nil, errors.New("archive/tar: FileInfo is nil")
}
fm := fi.Mode()
h := &Header{
@ -265,6 +688,12 @@ func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
h.Size = 0
h.Linkname = sys.Linkname
}
if sys.PAXRecords != nil {
h.PAXRecords = make(map[string]string)
for k, v := range sys.PAXRecords {
h.PAXRecords[k] = v
}
}
}
if sysStat != nil {
return h, sysStat(fi, h)
@ -282,3 +711,10 @@ func isHeaderOnlyType(flag byte) bool {
return false
}
}
func min(a, b int64) int64 {
if a < b {
return a
}
return b
}

174
libgo/go/archive/tar/format.go
View file

@ -4,38 +4,133 @@
package tar
import "strings"
// Format represents the tar archive format.
//
// The original tar format was introduced in Unix V7.
// Since then, there have been multiple competing formats attempting to
// standardize or extend the V7 format to overcome its limitations.
// The most common formats are the USTAR, PAX, and GNU formats,
// each with their own advantages and limitations.
//
// The following table captures the capabilities of each format:
//
// | USTAR | PAX | GNU
// ------------------+--------+-----------+----------
// Name | 256B | unlimited | unlimited
// Linkname | 100B | unlimited | unlimited
// Size | uint33 | unlimited | uint89
// Mode | uint21 | uint21 | uint57
// Uid/Gid | uint21 | unlimited | uint57
// Uname/Gname | 32B | unlimited | 32B
// ModTime | uint33 | unlimited | int89
// AccessTime | n/a | unlimited | int89
// ChangeTime | n/a | unlimited | int89
// Devmajor/Devminor | uint21 | uint21 | uint57
// ------------------+--------+-----------+----------
// string encoding | ASCII | UTF-8 | binary
// sub-second times | no | yes | no
// sparse files | no | yes | yes
//
// The table's upper portion shows the Header fields, where each format reports
// the maximum number of bytes allowed for each string field and
// the integer type used to store each numeric field
// (where timestamps are stored as the number of seconds since the Unix epoch).
//
// The table's lower portion shows specialized features of each format,
// such as supported string encodings, support for sub-second timestamps,
// or support for sparse files.
//
// The Writer currently provides no support for sparse files.
type Format int
// Constants to identify various tar formats.
const (
// The format is unknown.
formatUnknown = (1 << iota) / 2 // Sequence of 0, 1, 2, 4, 8, etc...
// Deliberately hide the meaning of constants from public API.
_ Format = (1 << iota) / 4 // Sequence of 0, 0, 1, 2, 4, 8, etc...
// FormatUnknown indicates that the format is unknown.
FormatUnknown
// The format of the original Unix V7 tar tool prior to standardization.
formatV7
// The old and new GNU formats, which are incompatible with USTAR.
// This does cover the old GNU sparse extension.
// This does not cover the GNU sparse extensions using PAX headers,
// versions 0.0, 0.1, and 1.0; these fall under the PAX format.
formatGNU
// FormatUSTAR represents the USTAR header format defined in POSIX.1-1988.
//
// While this format is compatible with most tar readers,
// the format has several limitations making it unsuitable for some usages.
// Most notably, it cannot support sparse files, files larger than 8GiB,
// filenames larger than 256 characters, and non-ASCII filenames.
//
// Reference:
// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06
FormatUSTAR
// FormatPAX represents the PAX header format defined in POSIX.1-2001.
//
// PAX extends USTAR by writing a special file with Typeflag TypeXHeader
// preceding the original header. This file contains a set of key-value
// records, which are used to overcome USTAR's shortcomings, in addition to
// providing the ability to have sub-second resolution for timestamps.
//
// Some newer formats add their own extensions to PAX by defining their
// own keys and assigning certain semantic meaning to the associated values.
// For example, sparse file support in PAX is implemented using keys
// defined by the GNU manual (e.g., "GNU.sparse.map").
//
// Reference:
// http://pubs.opengroup.org/onlinepubs/009695399/utilities/pax.html
FormatPAX
// FormatGNU represents the GNU header format.
//
// The GNU header format is older than the USTAR and PAX standards and
// is not compatible with them. The GNU format supports
// arbitrary file sizes, filenames of arbitrary encoding and length,
// sparse files, and other features.
//
// It is recommended that PAX be chosen over GNU unless the target
// application can only parse GNU formatted archives.
//
// Reference:
// http://www.gnu.org/software/tar/manual/html_node/Standard.html
FormatGNU
// Schily's tar format, which is incompatible with USTAR.
// This does not cover STAR extensions to the PAX format; these fall under
// the PAX format.
formatSTAR
// USTAR is the former standardization of tar defined in POSIX.1-1988.
// This is incompatible with the GNU and STAR formats.
formatUSTAR
// PAX is the latest standardization of tar defined in POSIX.1-2001.
// This is an extension of USTAR and is "backwards compatible" with it.
//
// Some newer formats add their own extensions to PAX, such as GNU sparse
// files and SCHILY extended attributes. Since they are backwards compatible
// with PAX, they will be labelled as "PAX".
formatPAX
formatMax
)
func (f Format) has(f2 Format) bool { return f&f2 != 0 }
func (f *Format) mayBe(f2 Format) { *f |= f2 }
func (f *Format) mayOnlyBe(f2 Format) { *f &= f2 }
func (f *Format) mustNotBe(f2 Format) { *f &^= f2 }
var formatNames = map[Format]string{
formatV7: "V7", FormatUSTAR: "USTAR", FormatPAX: "PAX", FormatGNU: "GNU", formatSTAR: "STAR",
}
func (f Format) String() string {
var ss []string
for f2 := Format(1); f2 < formatMax; f2 <<= 1 {
if f.has(f2) {
ss = append(ss, formatNames[f2])
}
}
switch len(ss) {
case 0:
return "<unknown>"
case 1:
return ss[0]
default:
return "(" + strings.Join(ss, " | ") + ")"
}
}
// Magics used to identify various formats.
const (
magicGNU, versionGNU = "ustar ", " \x00"
@ -50,6 +145,12 @@ const (
prefixSize = 155 // Max length of the prefix field in USTAR format
)
// blockPadding computes the number of bytes needed to pad offset up to the
// nearest block edge where 0 <= n < blockSize.
func blockPadding(offset int64) (n int64) {
return -offset & (blockSize - 1)
}
var zeroBlock block
type block [blockSize]byte
@ -63,14 +164,14 @@ func (b *block) Sparse() sparseArray { return (sparseArray)(b[:]) }
// GetFormat checks that the block is a valid tar header based on the checksum.
// It then attempts to guess the specific format based on magic values.
// If the checksum fails, then formatUnknown is returned.
func (b *block) GetFormat() (format int) {
// If the checksum fails, then FormatUnknown is returned.
func (b *block) GetFormat() Format {
// Verify checksum.
var p parser
value := p.parseOctal(b.V7().Chksum())
chksum1, chksum2 := b.ComputeChecksum()
if p.err != nil || (value != chksum1 && value != chksum2) {
return formatUnknown
return FormatUnknown
}
// Guess the magic values.
@ -81,9 +182,9 @@ func (b *block) GetFormat() (format int) {
case magic == magicUSTAR && trailer == trailerSTAR:
return formatSTAR
case magic == magicUSTAR:
return formatUSTAR
return FormatUSTAR | FormatPAX
case magic == magicGNU && version == versionGNU:
return formatGNU
return FormatGNU
default:
return formatV7
}
@ -91,19 +192,19 @@ func (b *block) GetFormat() (format int) {
// SetFormat writes the magic values necessary for specified format
// and then updates the checksum accordingly.
func (b *block) SetFormat(format int) {
func (b *block) SetFormat(format Format) {
// Set the magic values.
switch format {
case formatV7:
switch {
case format.has(formatV7):
// Do nothing.
case formatGNU:
case format.has(FormatGNU):
copy(b.GNU().Magic(), magicGNU)
copy(b.GNU().Version(), versionGNU)
case formatSTAR:
case format.has(formatSTAR):
copy(b.STAR().Magic(), magicUSTAR)
copy(b.STAR().Version(), versionUSTAR)
copy(b.STAR().Trailer(), trailerSTAR)
case formatUSTAR, formatPAX:
case format.has(FormatUSTAR | FormatPAX):
copy(b.USTAR().Magic(), magicUSTAR)
copy(b.USTAR().Version(), versionUSTAR)
default:
@ -128,12 +229,17 @@ func (b *block) ComputeChecksum() (unsigned, signed int64) {
if 148 <= i && i < 156 {
c = ' ' // Treat the checksum field itself as all spaces.
}
unsigned += int64(uint8(c))
unsigned += int64(c)
signed += int64(int8(c))
}
return unsigned, signed
}
// Reset clears the block with all zeros.
func (b *block) Reset() {
*b = block{}
}
type headerV7 [blockSize]byte
func (h *headerV7) Name() []byte { return h[000:][:100] }
@ -187,11 +293,11 @@ func (h *headerUSTAR) Prefix() []byte { return h[345:][:155] }
type sparseArray []byte
func (s sparseArray) Entry(i int) sparseNode { return (sparseNode)(s[i*24:]) }
func (s sparseArray) Entry(i int) sparseElem { return (sparseElem)(s[i*24:]) }
func (s sparseArray) IsExtended() []byte { return s[24*s.MaxEntries():][:1] }
func (s sparseArray) MaxEntries() int { return len(s) / 24 }
type sparseNode []byte
type sparseElem []byte
func (s sparseNode) Offset() []byte { return s[00:][:12] }
func (s sparseNode) NumBytes() []byte { return s[12:][:12] }
func (s sparseElem) Offset() []byte { return s[00:][:12] }
func (s sparseElem) Length() []byte { return s[12:][:12] }

865
libgo/go/archive/tar/reader.go
View file

File diff suppressed because it is too large Load diff

1428
libgo/go/archive/tar/reader_test.go
View file

File diff suppressed because it is too large Load diff

0
libgo/go/archive/tar/stat_atim.go → libgo/go/archive/tar/stat_actime1.go
View file

0
libgo/go/archive/tar/stat_atimespec.go → libgo/go/archive/tar/stat_actime2.go
View file

72
libgo/go/archive/tar/stat_unix.go
View file

@ -8,6 +8,10 @@ package tar
import (
"os"
"os/user"
"runtime"
"strconv"
"sync"
"syscall"
)
@ -15,6 +19,10 @@ func init() {
sysStat = statUnix
}
// userMap and groupMap caches UID and GID lookups for performance reasons.
// The downside is that renaming uname or gname by the OS never takes effect.
var userMap, groupMap sync.Map // map[int]string
func statUnix(fi os.FileInfo, h *Header) error {
sys, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
@ -22,11 +30,67 @@ func statUnix(fi os.FileInfo, h *Header) error {
}
h.Uid = int(sys.Uid)
h.Gid = int(sys.Gid)
// TODO(bradfitz): populate username & group. os/user
// doesn't cache LookupId lookups, and lacks group
// lookup functions.
// Best effort at populating Uname and Gname.
// The os/user functions may fail for any number of reasons
// (not implemented on that platform, cgo not enabled, etc).
if u, ok := userMap.Load(h.Uid); ok {
h.Uname = u.(string)
} else if u, err := user.LookupId(strconv.Itoa(h.Uid)); err == nil {
h.Uname = u.Username
userMap.Store(h.Uid, h.Uname)
}
if g, ok := groupMap.Load(h.Gid); ok {
h.Gname = g.(string)
} else if g, err := user.LookupGroupId(strconv.Itoa(h.Gid)); err == nil {
h.Gname = g.Name
groupMap.Store(h.Gid, h.Gname)
}
h.AccessTime = statAtime(sys)
h.ChangeTime = statCtime(sys)
// TODO(bradfitz): major/minor device numbers?
// Best effort at populating Devmajor and Devminor.
if h.Typeflag == TypeChar || h.Typeflag == TypeBlock {
dev := uint64(sys.Rdev) // May be int32 or uint32
switch runtime.GOOS {
case "linux":
// Copied from golang.org/x/sys/unix/dev_linux.go.
major := uint32((dev & 0x00000000000fff00) >> 8)
major |= uint32((dev & 0xfffff00000000000) >> 32)
minor := uint32((dev & 0x00000000000000ff) >> 0)
minor |= uint32((dev & 0x00000ffffff00000) >> 12)
h.Devmajor, h.Devminor = int64(major), int64(minor)
case "darwin":
// Copied from golang.org/x/sys/unix/dev_darwin.go.
major := uint32((dev >> 24) & 0xff)
minor := uint32(dev & 0xffffff)
h.Devmajor, h.Devminor = int64(major), int64(minor)
case "dragonfly":
// Copied from golang.org/x/sys/unix/dev_dragonfly.go.
major := uint32((dev >> 8) & 0xff)
minor := uint32(dev & 0xffff00ff)
h.Devmajor, h.Devminor = int64(major), int64(minor)
case "freebsd":
// Copied from golang.org/x/sys/unix/dev_freebsd.go.
major := uint32((dev >> 8) & 0xff)
minor := uint32(dev & 0xffff00ff)
h.Devmajor, h.Devminor = int64(major), int64(minor)
case "netbsd":
// Copied from golang.org/x/sys/unix/dev_netbsd.go.
major := uint32((dev & 0x000fff00) >> 8)
minor := uint32((dev & 0x000000ff) >> 0)
minor |= uint32((dev & 0xfff00000) >> 12)
h.Devmajor, h.Devminor = int64(major), int64(minor)
case "openbsd":
// Copied from golang.org/x/sys/unix/dev_openbsd.go.
major := uint32((dev & 0x0000ff00) >> 8)
minor := uint32((dev & 0x000000ff) >> 0)
minor |= uint32((dev & 0xffff0000) >> 8)
h.Devmajor, h.Devminor = int64(major), int64(minor)
default:
// TODO: Implement solaris (see https://golang.org/issue/8106)
}
}
return nil
}

130
libgo/go/archive/tar/strconv.go
View file

@ -12,26 +12,34 @@ import (
"time"
)
// hasNUL reports whether the NUL character exists within s.
func hasNUL(s string) bool {
return strings.IndexByte(s, 0) >= 0
}
// isASCII reports whether the input is an ASCII C-style string.
func isASCII(s string) bool {
for _, c := range s {
if c >= 0x80 {
if c >= 0x80 || c == 0x00 {
return false
}
}
return true
}
// toASCII converts the input to an ASCII C-style string.
// This a best effort conversion, so invalid characters are dropped.
func toASCII(s string) string {
if isASCII(s) {
return s
}
var buf bytes.Buffer
b := make([]byte, 0, len(s))
for _, c := range s {
if c < 0x80 {
buf.WriteByte(byte(c))
if c < 0x80 && c != 0x00 {
b = append(b, byte(c))
}
}
return buf.String()
return string(b)
}
type parser struct {
@ -45,23 +53,28 @@ type formatter struct {
// parseString parses bytes as a NUL-terminated C-style string.
// If a NUL byte is not found then the whole slice is returned as a string.
func (*parser) parseString(b []byte) string {
n := 0
for n < len(b) && b[n] != 0 {
n++
if i := bytes.IndexByte(b, 0); i >= 0 {
return string(b[:i])
}
return string(b[0:n])
return string(b)
}
// Write s into b, terminating it with a NUL if there is room.
// formatString copies s into b, NUL-terminating if possible.
func (f *formatter) formatString(b []byte, s string) {
if len(s) > len(b) {
f.err = ErrFieldTooLong
return
}
ascii := toASCII(s)
copy(b, ascii)
if len(ascii) < len(b) {
b[len(ascii)] = 0
copy(b, s)
if len(s) < len(b) {
b[len(s)] = 0
}
// Some buggy readers treat regular files with a trailing slash
// in the V7 path field as a directory even though the full path
// recorded elsewhere (e.g., via PAX record) contains no trailing slash.
if len(s) > len(b) && b[len(b)-1] == '/' {
n := len(strings.TrimRight(s[:len(b)], "/"))
b[n] = 0 // Replace trailing slash with NUL terminator
}
}
@ -73,7 +86,7 @@ func (f *formatter) formatString(b []byte, s string) {
// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is
// equivalent to the sign bit in two's complement form.
func fitsInBase256(n int, x int64) bool {
var binBits = uint(n-1) * 8
binBits := uint(n-1) * 8
return n >= 9 || (x >= -1<<binBits && x < 1<<binBits)
}
@ -121,8 +134,14 @@ func (p *parser) parseNumeric(b []byte) int64 {
return p.parseOctal(b)
}
// Write x into b, as binary (GNUtar/star extension).
// formatNumeric encodes x into b using base-8 (octal) encoding if possible.
// Otherwise it will attempt to use base-256 (binary) encoding.
func (f *formatter) formatNumeric(b []byte, x int64) {
if fitsInOctal(len(b), x) {
f.formatOctal(b, x)
return
}
if fitsInBase256(len(b), x) {
for i := len(b) - 1; i >= 0; i-- {
b[i] = byte(x)
@ -155,6 +174,11 @@ func (p *parser) parseOctal(b []byte) int64 {
}
func (f *formatter) formatOctal(b []byte, x int64) {
if !fitsInOctal(len(b), x) {
x = 0 // Last resort, just write zero
f.err = ErrFieldTooLong
}
s := strconv.FormatInt(x, 8)
// Add leading zeros, but leave room for a NUL.
if n := len(b) - len(s) - 1; n > 0 {
@ -163,6 +187,13 @@ func (f *formatter) formatOctal(b []byte, x int64) {
f.formatString(b, s)
}
// fitsInOctal reports whether the integer x fits in a field n-bytes long
// using octal encoding with the appropriate NUL terminator.
func fitsInOctal(n int, x int64) bool {
octBits := uint(n-1) * 3
return x >= 0 && (n >= 22 || x < 1<<octBits)
}
// parsePAXTime takes a string of the form %d.%d as described in the PAX
// specification. Note that this implementation allows for negative timestamps,
// which is allowed for by the PAX specification, but not always portable.
@ -195,19 +226,32 @@ func parsePAXTime(s string) (time.Time, error) {
}
nsecs, _ := strconv.ParseInt(sn, 10, 64) // Must succeed
if len(ss) > 0 && ss[0] == '-' {
return time.Unix(secs, -1*int64(nsecs)), nil // Negative correction
return time.Unix(secs, -1*nsecs), nil // Negative correction
}
return time.Unix(secs, int64(nsecs)), nil
return time.Unix(secs, nsecs), nil
}
// TODO(dsnet): Implement formatPAXTime.
// formatPAXTime converts ts into a time of the form %d.%d as described in the
// PAX specification. This function is capable of negative timestamps.
func formatPAXTime(ts time.Time) (s string) {
secs, nsecs := ts.Unix(), ts.Nanosecond()
if nsecs == 0 {
return strconv.FormatInt(secs, 10)
}
// If seconds is negative, then perform correction.
sign := ""
if secs < 0 {
sign = "-" // Remember sign
secs = -(secs + 1) // Add a second to secs
nsecs = -(nsecs - 1E9) // Take that second away from nsecs
}
return strings.TrimRight(fmt.Sprintf("%s%d.%09d", sign, secs, nsecs), "0")
}
// parsePAXRecord parses the input PAX record string into a key-value pair.
// If parsing is successful, it will slice off the currently read record and
// return the remainder as r.
//
// A PAX record is of the following form:
// "%d %s=%s\n" % (size, key, value)
func parsePAXRecord(s string) (k, v, r string, err error) {
// The size field ends at the first space.
sp := strings.IndexByte(s, ' ')
@ -232,21 +276,51 @@ func parsePAXRecord(s string) (k, v, r string, err error) {
if eq == -1 {
return "", "", s, ErrHeader
}
return rec[:eq], rec[eq+1:], rem, nil
k, v = rec[:eq], rec[eq+1:]
if !validPAXRecord(k, v) {
return "", "", s, ErrHeader
}
return k, v, rem, nil
}
// formatPAXRecord formats a single PAX record, prefixing it with the
// appropriate length.
func formatPAXRecord(k, v string) string {
func formatPAXRecord(k, v string) (string, error) {
if !validPAXRecord(k, v) {
return "", ErrHeader
}
const padding = 3 // Extra padding for ' ', '=', and '\n'
size := len(k) + len(v) + padding
size += len(strconv.Itoa(size))
record := fmt.Sprintf("%d %s=%s\n", size, k, v)
record := strconv.Itoa(size) + " " + k + "=" + v + "\n"
// Final adjustment if adding size field increased the record size.
if len(record) != size {
size = len(record)
record = fmt.Sprintf("%d %s=%s\n", size, k, v)
record = strconv.Itoa(size) + " " + k + "=" + v + "\n"
}
return record, nil
}
// validPAXRecord reports whether the key-value pair is valid where each
// record is formatted as:
// "%d %s=%s\n" % (size, key, value)
//
// Keys and values should be UTF-8, but the number of bad writers out there
// forces us to be a more liberal.
// Thus, we only reject all keys with NUL, and only reject NULs in values
// for the PAX version of the USTAR string fields.
// The key must not contain an '=' character.
func validPAXRecord(k, v string) bool {
if k == "" || strings.IndexByte(k, '=') >= 0 {
return false
}
switch k {
case paxPath, paxLinkpath, paxUname, paxGname:
return !hasNUL(v)
default:
return !hasNUL(k)
}
return record
}

139
libgo/go/archive/tar/strconv_test.go
View file

@ -110,6 +110,25 @@ func TestFormatNumeric(t *testing.T) {
want string
ok bool
}{
// Test base-8 (octal) encoded values.
{0, "0\x00", true},
{7, "7\x00", true},
{8, "\x80\x08", true},
{077, "77\x00", true},
{0100, "\x80\x00\x40", true},
{0, "0000000\x00", true},
{0123, "0000123\x00", true},
{07654321, "7654321\x00", true},
{07777777, "7777777\x00", true},
{010000000, "\x80\x00\x00\x00\x00\x20\x00\x00", true},
{0, "00000000000\x00", true},
{000001234567, "00001234567\x00", true},
{076543210321, "76543210321\x00", true},
{012345670123, "12345670123\x00", true},
{077777777777, "77777777777\x00", true},
{0100000000000, "\x80\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00", true},
{math.MaxInt64, "777777777777777777777\x00", true},
// Test base-256 (binary) encoded values.
{-1, "\xff", true},
{-1, "\xff\xff", true},
@ -155,6 +174,45 @@ func TestFormatNumeric(t *testing.T) {
}
}
func TestFitsInOctal(t *testing.T) {
vectors := []struct {
input int64
width int
ok bool
}{
{-1, 1, false},
{-1, 2, false},
{-1, 3, false},
{0, 1, true},
{0 + 1, 1, false},
{0, 2, true},
{07, 2, true},
{07 + 1, 2, false},
{0, 4, true},
{0777, 4, true},
{0777 + 1, 4, false},
{0, 8, true},
{07777777, 8, true},
{07777777 + 1, 8, false},
{0, 12, true},
{077777777777, 12, true},
{077777777777 + 1, 12, false},
{math.MaxInt64, 22, true},
{012345670123, 12, true},
{01564164, 12, true},
{-012345670123, 12, false},
{-01564164, 12, false},
{-1564164, 30, false},
}
for _, v := range vectors {
ok := fitsInOctal(v.width, v.input)
if ok != v.ok {
t.Errorf("checkOctal(%d, %d): got %v, want %v", v.input, v.width, ok, v.ok)
}
}
}
func TestParsePAXTime(t *testing.T) {
vectors := []struct {
in string
@ -236,6 +294,51 @@ func TestParsePAXTime(t *testing.T) {
}
}
func TestFormatPAXTime(t *testing.T) {
vectors := []struct {
sec, nsec int64
want string
}{
{1350244992, 0, "1350244992"},
{1350244992, 300000000, "1350244992.3"},
{1350244992, 23960100, "1350244992.0239601"},
{1350244992, 23960108, "1350244992.023960108"},
{+1, +1E9 - 1E0, "1.999999999"},
{+1, +1E9 - 1E3, "1.999999"},
{+1, +1E9 - 1E6, "1.999"},
{+1, +0E0 - 0E0, "1"},
{+1, +1E6 - 0E0, "1.001"},
{+1, +1E3 - 0E0, "1.000001"},
{+1, +1E0 - 0E0, "1.000000001"},
{0, 1E9 - 1E0, "0.999999999"},
{0, 1E9 - 1E3, "0.999999"},
{0, 1E9 - 1E6, "0.999"},
{0, 0E0, "0"},
{0, 1E6 + 0E0, "0.001"},
{0, 1E3 + 0E0, "0.000001"},
{0, 1E0 + 0E0, "0.000000001"},
{-1, -1E9 + 1E0, "-1.999999999"},
{-1, -1E9 + 1E3, "-1.999999"},
{-1, -1E9 + 1E6, "-1.999"},
{-1, -0E0 + 0E0, "-1"},
{-1, -1E6 + 0E0, "-1.001"},
{-1, -1E3 + 0E0, "-1.000001"},
{-1, -1E0 + 0E0, "-1.000000001"},
{-1350244992, 0, "-1350244992"},
{-1350244992, -300000000, "-1350244992.3"},
{-1350244992, -23960100, "-1350244992.0239601"},
{-1350244992, -23960108, "-1350244992.023960108"},
}
for _, v := range vectors {
got := formatPAXTime(time.Unix(v.sec, v.nsec))
if got != v.want {
t.Errorf("formatPAXTime(%ds, %dns): got %q, want %q",
v.sec, v.nsec, got, v.want)
}
}
}
func TestParsePAXRecord(t *testing.T) {
medName := strings.Repeat("CD", 50)
longName := strings.Repeat("AB", 100)
@ -256,7 +359,7 @@ func TestParsePAXRecord(t *testing.T) {
{"18 foo=b=\nar=\n==\x00\n", "", "foo", "b=\nar=\n==\x00", true},
{"27 foo=hello9 foo=ba\nworld\n", "", "foo", "hello9 foo=ba\nworld", true},
{"27 ☺☻☹=日a本b語ç\nmeow mix", "meow mix", "☺☻☹", "日a本b語ç", true},
{"17 \x00hello=\x00world\n", "", "\x00hello", "\x00world", true},
{"17 \x00hello=\x00world\n", "17 \x00hello=\x00world\n", "", "", false},
{"1 k=1\n", "1 k=1\n", "", "", false},
{"6 k~1\n", "6 k~1\n", "", "", false},
{"6_k=1\n", "6_k=1\n", "", "", false},
@ -296,21 +399,33 @@ func TestFormatPAXRecord(t *testing.T) {
inKey string
inVal string
want string
ok bool
}{
{"k", "v", "6 k=v\n"},
{"path", "/etc/hosts", "19 path=/etc/hosts\n"},
{"path", longName, "210 path=" + longName + "\n"},
{"path", medName, "110 path=" + medName + "\n"},
{"foo", "ba", "9 foo=ba\n"},
{"foo", "bar", "11 foo=bar\n"},
{"foo", "b=\nar=\n==\x00", "18 foo=b=\nar=\n==\x00\n"},
{"foo", "hello9 foo=ba\nworld", "27 foo=hello9 foo=ba\nworld\n"},
{"☺☻☹", "日a本b語ç", "27 ☺☻☹=日a本b語ç\n"},
{"\x00hello", "\x00world", "17 \x00hello=\x00world\n"},
{"k", "v", "6 k=v\n", true},
{"path", "/etc/hosts", "19 path=/etc/hosts\n", true},
{"path", longName, "210 path=" + longName + "\n", true},
{"path", medName, "110 path=" + medName + "\n", true},
{"foo", "ba", "9 foo=ba\n", true},
{"foo", "bar", "11 foo=bar\n", true},
{"foo", "b=\nar=\n==\x00", "18 foo=b=\nar=\n==\x00\n", true},
{"foo", "hello9 foo=ba\nworld", "27 foo=hello9 foo=ba\nworld\n", true},
{"☺☻☹", "日a本b語ç", "27 ☺☻☹=日a本b語ç\n", true},
{"xhello", "\x00world", "17 xhello=\x00world\n", true},
{"path", "null\x00", "", false},
{"null\x00", "value", "", false},
{paxSchilyXattr + "key", "null\x00", "26 SCHILY.xattr.key=null\x00\n", true},
}
for _, v := range vectors {
got := formatPAXRecord(v.inKey, v.inVal)
got, err := formatPAXRecord(v.inKey, v.inVal)
ok := (err == nil)
if ok != v.ok {
if v.ok {
t.Errorf("formatPAXRecord(%q, %q): got format failure, want success", v.inKey, v.inVal)
} else {
t.Errorf("formatPAXRecord(%q, %q): got format success, want failure", v.inKey, v.inVal)
}
}
if got != v.want {
t.Errorf("formatPAXRecord(%q, %q): got %q, want %q",
v.inKey, v.inVal, got, v.want)

541
libgo/go/archive/tar/tar_test.go
View file

@ -6,8 +6,12 @@ package tar
import (
"bytes"
"errors"
"fmt"
"internal/testenv"
"io"
"io/ioutil"
"math"
"os"
"path"
"path/filepath"
@ -17,6 +21,193 @@ import (
"time"
)
type testError struct{ error }
type fileOps []interface{} // []T where T is (string | int64)
// testFile is an io.ReadWriteSeeker where the IO operations performed
// on it must match the list of operations in ops.
type testFile struct {
ops fileOps
pos int64
}
func (f *testFile) Read(b []byte) (int, error) {
if len(b) == 0 {
return 0, nil
}
if len(f.ops) == 0 {
return 0, io.EOF
}
s, ok := f.ops[0].(string)
if !ok {
return 0, errors.New("unexpected Read operation")
}
n := copy(b, s)
if len(s) > n {
f.ops[0] = s[n:]
} else {
f.ops = f.ops[1:]
}
f.pos += int64(len(b))
return n, nil
}
func (f *testFile) Write(b []byte) (int, error) {
if len(b) == 0 {
return 0, nil
}
if len(f.ops) == 0 {
return 0, errors.New("unexpected Write operation")
}
s, ok := f.ops[0].(string)
if !ok {
return 0, errors.New("unexpected Write operation")
}
if !strings.HasPrefix(s, string(b)) {
return 0, testError{fmt.Errorf("got Write(%q), want Write(%q)", b, s)}
}
if len(s) > len(b) {
f.ops[0] = s[len(b):]
} else {
f.ops = f.ops[1:]
}
f.pos += int64(len(b))
return len(b), nil
}
func (f *testFile) Seek(pos int64, whence int) (int64, error) {
if pos == 0 && whence == io.SeekCurrent {
return f.pos, nil
}
if len(f.ops) == 0 {
return 0, errors.New("unexpected Seek operation")
}
s, ok := f.ops[0].(int64)
if !ok {
return 0, errors.New("unexpected Seek operation")
}
if s != pos || whence != io.SeekCurrent {
return 0, testError{fmt.Errorf("got Seek(%d, %d), want Seek(%d, %d)", pos, whence, s, io.SeekCurrent)}
}
f.pos += s
f.ops = f.ops[1:]
return f.pos, nil
}
func equalSparseEntries(x, y []sparseEntry) bool {
return (len(x) == 0 && len(y) == 0) || reflect.DeepEqual(x, y)
}
func TestSparseEntries(t *testing.T) {
vectors := []struct {
in []sparseEntry
size int64
wantValid bool // Result of validateSparseEntries
wantAligned []sparseEntry // Result of alignSparseEntries
wantInverted []sparseEntry // Result of invertSparseEntries
}{{
in: []sparseEntry{}, size: 0,
wantValid: true,
wantInverted: []sparseEntry{{0, 0}},
}, {
in: []sparseEntry{}, size: 5000,
wantValid: true,
wantInverted: []sparseEntry{{0, 5000}},
}, {
in: []sparseEntry{{0, 5000}}, size: 5000,
wantValid: true,
wantAligned: []sparseEntry{{0, 5000}},
wantInverted: []sparseEntry{{5000, 0}},
}, {
in: []sparseEntry{{1000, 4000}}, size: 5000,
wantValid: true,
wantAligned: []sparseEntry{{1024, 3976}},
wantInverted: []sparseEntry{{0, 1000}, {5000, 0}},
}, {
in: []sparseEntry{{0, 3000}}, size: 5000,
wantValid: true,
wantAligned: []sparseEntry{{0, 2560}},
wantInverted: []sparseEntry{{3000, 2000}},
}, {
in: []sparseEntry{{3000, 2000}}, size: 5000,
wantValid: true,
wantAligned: []sparseEntry{{3072, 1928}},
wantInverted: []sparseEntry{{0, 3000}, {5000, 0}},
}, {
in: []sparseEntry{{2000, 2000}}, size: 5000,
wantValid: true,
wantAligned: []sparseEntry{{2048, 1536}},
wantInverted: []sparseEntry{{0, 2000}, {4000, 1000}},
}, {
in: []sparseEntry{{0, 2000}, {8000, 2000}}, size: 10000,
wantValid: true,
wantAligned: []sparseEntry{{0, 1536}, {8192, 1808}},
wantInverted: []sparseEntry{{2000, 6000}, {10000, 0}},
}, {
in: []sparseEntry{{0, 2000}, {2000, 2000}, {4000, 0}, {4000, 3000}, {7000, 1000}, {8000, 0}, {8000, 2000}}, size: 10000,
wantValid: true,
wantAligned: []sparseEntry{{0, 1536}, {2048, 1536}, {4096, 2560}, {7168, 512}, {8192, 1808}},
wantInverted: []sparseEntry{{10000, 0}},
}, {
in: []sparseEntry{{0, 0}, {1000, 0}, {2000, 0}, {3000, 0}, {4000, 0}, {5000, 0}}, size: 5000,
wantValid: true,
wantInverted: []sparseEntry{{0, 5000}},
}, {
in: []sparseEntry{{1, 0}}, size: 0,
wantValid: false,
}, {
in: []sparseEntry{{-1, 0}}, size: 100,
wantValid: false,
}, {
in: []sparseEntry{{0, -1}}, size: 100,
wantValid: false,
}, {
in: []sparseEntry{{0, 0}}, size: -100,
wantValid: false,
}, {
in: []sparseEntry{{math.MaxInt64, 3}, {6, -5}}, size: 35,
wantValid: false,
}, {
in: []sparseEntry{{1, 3}, {6, -5}}, size: 35,
wantValid: false,
}, {
in: []sparseEntry{{math.MaxInt64, math.MaxInt64}}, size: math.MaxInt64,
wantValid: false,
}, {
in: []sparseEntry{{3, 3}}, size: 5,
wantValid: false,
}, {
in: []sparseEntry{{2, 0}, {1, 0}, {0, 0}}, size: 3,
wantValid: false,
}, {
in: []sparseEntry{{1, 3}, {2, 2}}, size: 10,
wantValid: false,
}}
for i, v := range vectors {
gotValid := validateSparseEntries(v.in, v.size)
if gotValid != v.wantValid {
t.Errorf("test %d, validateSparseEntries() = %v, want %v", i, gotValid, v.wantValid)
}
if !v.wantValid {
continue
}
gotAligned := alignSparseEntries(append([]sparseEntry{}, v.in...), v.size)
if !equalSparseEntries(gotAligned, v.wantAligned) {
t.Errorf("test %d, alignSparseEntries():\ngot %v\nwant %v", i, gotAligned, v.wantAligned)
}
gotInverted := invertSparseEntries(append([]sparseEntry{}, v.in...), v.size)
if !equalSparseEntries(gotInverted, v.wantInverted) {
t.Errorf("test %d, inverseSparseEntries():\ngot %v\nwant %v", i, gotInverted, v.wantInverted)
}
}
}
func TestFileInfoHeader(t *testing.T) {
fi, err := os.Stat("testdata/small.txt")
if err != nil {
@ -109,15 +300,12 @@ func TestRoundTrip(t *testing.T) {
var b bytes.Buffer
tw := NewWriter(&b)
hdr := &Header{
Name: "file.txt",
Uid: 1 << 21, // too big for 8 octal digits
Size: int64(len(data)),
// AddDate to strip monotonic clock reading,
// and Round to discard sub-second precision,
// both of which are not included in the tar header
// and would otherwise break the round-trip check
// below.
ModTime: time.Now().AddDate(0, 0, 0).Round(1 * time.Second),
Name: "file.txt",
Uid: 1 << 21, // Too big for 8 octal digits
Size: int64(len(data)),
ModTime: time.Now().Round(time.Second),
PAXRecords: map[string]string{"uid": "2097152"},
Format: FormatPAX,
}
if err := tw.WriteHeader(hdr); err != nil {
t.Fatalf("tw.WriteHeader: %v", err)
@ -329,3 +517,338 @@ func TestHeaderRoundTrip(t *testing.T) {
}
}
}
func TestHeaderAllowedFormats(t *testing.T) {
vectors := []struct {
header *Header // Input header
paxHdrs map[string]string // Expected PAX headers that may be needed
formats Format // Expected formats that can encode the header
}{{
header: &Header{},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{Size: 077777777777},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{Size: 077777777777, Format: FormatUSTAR},
formats: FormatUSTAR,
}, {
header: &Header{Size: 077777777777, Format: FormatPAX},
formats: FormatUSTAR | FormatPAX,
}, {
header: &Header{Size: 077777777777, Format: FormatGNU},
formats: FormatGNU,
}, {
header: &Header{Size: 077777777777 + 1},
paxHdrs: map[string]string{paxSize: "8589934592"},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{Size: 077777777777 + 1, Format: FormatPAX},
paxHdrs: map[string]string{paxSize: "8589934592"},
formats: FormatPAX,
}, {
header: &Header{Size: 077777777777 + 1, Format: FormatGNU},
paxHdrs: map[string]string{paxSize: "8589934592"},
formats: FormatGNU,
}, {
header: &Header{Mode: 07777777},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{Mode: 07777777 + 1},
formats: FormatGNU,
}, {
header: &Header{Devmajor: -123},
formats: FormatGNU,
}, {
header: &Header{Devmajor: 1<<56 - 1},
formats: FormatGNU,
}, {
header: &Header{Devmajor: 1 << 56},
formats: FormatUnknown,
}, {
header: &Header{Devmajor: -1 << 56},
formats: FormatGNU,
}, {
header: &Header{Devmajor: -1<<56 - 1},
formats: FormatUnknown,
}, {
header: &Header{Name: "用戶名", Devmajor: -1 << 56},
formats: FormatGNU,
}, {
header: &Header{Size: math.MaxInt64},
paxHdrs: map[string]string{paxSize: "9223372036854775807"},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{Size: math.MinInt64},
paxHdrs: map[string]string{paxSize: "-9223372036854775808"},
formats: FormatUnknown,
}, {
header: &Header{Uname: "0123456789abcdef0123456789abcdef"},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{Uname: "0123456789abcdef0123456789abcdefx"},
paxHdrs: map[string]string{paxUname: "0123456789abcdef0123456789abcdefx"},
formats: FormatPAX,
}, {
header: &Header{Name: "foobar"},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{Name: strings.Repeat("a", nameSize)},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{Name: strings.Repeat("a", nameSize+1)},
paxHdrs: map[string]string{paxPath: strings.Repeat("a", nameSize+1)},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{Linkname: "用戶名"},
paxHdrs: map[string]string{paxLinkpath: "用戶名"},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{Linkname: strings.Repeat("用戶名\x00", nameSize)},
paxHdrs: map[string]string{paxLinkpath: strings.Repeat("用戶名\x00", nameSize)},
formats: FormatUnknown,
}, {
header: &Header{Linkname: "\x00hello"},
paxHdrs: map[string]string{paxLinkpath: "\x00hello"},
formats: FormatUnknown,
}, {
header: &Header{Uid: 07777777},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{Uid: 07777777 + 1},
paxHdrs: map[string]string{paxUid: "2097152"},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{Xattrs: nil},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{Xattrs: map[string]string{"foo": "bar"}},
paxHdrs: map[string]string{paxSchilyXattr + "foo": "bar"},
formats: FormatPAX,
}, {
header: &Header{Xattrs: map[string]string{"foo": "bar"}, Format: FormatGNU},
paxHdrs: map[string]string{paxSchilyXattr + "foo": "bar"},
formats: FormatUnknown,
}, {
header: &Header{Xattrs: map[string]string{"用戶名": "\x00hello"}},
paxHdrs: map[string]string{paxSchilyXattr + "用戶名": "\x00hello"},
formats: FormatPAX,
}, {
header: &Header{Xattrs: map[string]string{"foo=bar": "baz"}},
formats: FormatUnknown,
}, {
header: &Header{Xattrs: map[string]string{"foo": ""}},
paxHdrs: map[string]string{paxSchilyXattr + "foo": ""},
formats: FormatPAX,
}, {
header: &Header{ModTime: time.Unix(0, 0)},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{ModTime: time.Unix(077777777777, 0)},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{ModTime: time.Unix(077777777777+1, 0)},
paxHdrs: map[string]string{paxMtime: "8589934592"},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{ModTime: time.Unix(math.MaxInt64, 0)},
paxHdrs: map[string]string{paxMtime: "9223372036854775807"},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{ModTime: time.Unix(math.MaxInt64, 0), Format: FormatUSTAR},
paxHdrs: map[string]string{paxMtime: "9223372036854775807"},
formats: FormatUnknown,
}, {
header: &Header{ModTime: time.Unix(-1, 0)},
paxHdrs: map[string]string{paxMtime: "-1"},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{ModTime: time.Unix(1, 500)},
paxHdrs: map[string]string{paxMtime: "1.0000005"},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{ModTime: time.Unix(1, 0)},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{ModTime: time.Unix(1, 0), Format: FormatPAX},
formats: FormatUSTAR | FormatPAX,
}, {
header: &Header{ModTime: time.Unix(1, 500), Format: FormatUSTAR},
paxHdrs: map[string]string{paxMtime: "1.0000005"},
formats: FormatUSTAR,
}, {
header: &Header{ModTime: time.Unix(1, 500), Format: FormatPAX},
paxHdrs: map[string]string{paxMtime: "1.0000005"},
formats: FormatPAX,
}, {
header: &Header{ModTime: time.Unix(1, 500), Format: FormatGNU},
paxHdrs: map[string]string{paxMtime: "1.0000005"},
formats: FormatGNU,
}, {
header: &Header{ModTime: time.Unix(-1, 500)},
paxHdrs: map[string]string{paxMtime: "-0.9999995"},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{ModTime: time.Unix(-1, 500), Format: FormatGNU},
paxHdrs: map[string]string{paxMtime: "-0.9999995"},
formats: FormatGNU,
}, {
header: &Header{AccessTime: time.Unix(0, 0)},
paxHdrs: map[string]string{paxAtime: "0"},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{AccessTime: time.Unix(0, 0), Format: FormatUSTAR},
paxHdrs: map[string]string{paxAtime: "0"},
formats: FormatUnknown,
}, {
header: &Header{AccessTime: time.Unix(0, 0), Format: FormatPAX},
paxHdrs: map[string]string{paxAtime: "0"},
formats: FormatPAX,
}, {
header: &Header{AccessTime: time.Unix(0, 0), Format: FormatGNU},
paxHdrs: map[string]string{paxAtime: "0"},
formats: FormatGNU,
}, {
header: &Header{AccessTime: time.Unix(-123, 0)},
paxHdrs: map[string]string{paxAtime: "-123"},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{AccessTime: time.Unix(-123, 0), Format: FormatPAX},
paxHdrs: map[string]string{paxAtime: "-123"},
formats: FormatPAX,
}, {
header: &Header{ChangeTime: time.Unix(123, 456)},
paxHdrs: map[string]string{paxCtime: "123.000000456"},
formats: FormatPAX | FormatGNU,
}, {
header: &Header{ChangeTime: time.Unix(123, 456), Format: FormatUSTAR},
paxHdrs: map[string]string{paxCtime: "123.000000456"},
formats: FormatUnknown,
}, {
header: &Header{ChangeTime: time.Unix(123, 456), Format: FormatGNU},
paxHdrs: map[string]string{paxCtime: "123.000000456"},
formats: FormatGNU,
}, {
header: &Header{ChangeTime: time.Unix(123, 456), Format: FormatPAX},
paxHdrs: map[string]string{paxCtime: "123.000000456"},
formats: FormatPAX,
}, {
header: &Header{Name: "foo/", Typeflag: TypeDir},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}, {
header: &Header{Name: "foo/", Typeflag: TypeReg},
formats: FormatUnknown,
}, {
header: &Header{Name: "foo/", Typeflag: TypeSymlink},
formats: FormatUSTAR | FormatPAX | FormatGNU,
}}
for i, v := range vectors {
formats, paxHdrs, err := v.header.allowedFormats()
if formats != v.formats {
t.Errorf("test %d, allowedFormats(): got %v, want %v", i, formats, v.formats)
}
if formats&FormatPAX > 0 && !reflect.DeepEqual(paxHdrs, v.paxHdrs) && !(len(paxHdrs) == 0 && len(v.paxHdrs) == 0) {
t.Errorf("test %d, allowedFormats():\ngot %v\nwant %s", i, paxHdrs, v.paxHdrs)
}
if (formats != FormatUnknown) && (err != nil) {
t.Errorf("test %d, unexpected error: %v", i, err)
}
if (formats == FormatUnknown) && (err == nil) {
t.Errorf("test %d, got nil-error, want non-nil error", i)
}
}
}
func Benchmark(b *testing.B) {
type file struct {
hdr *Header
body []byte
}
vectors := []struct {
label string
files []file
}{{
"USTAR",
[]file{{
&Header{Name: "bar", Mode: 0640, Size: int64(3)},
[]byte("foo"),
}, {
&Header{Name: "world", Mode: 0640, Size: int64(5)},
[]byte("hello"),
}},
}, {
"GNU",
[]file{{
&Header{Name: "bar", Mode: 0640, Size: int64(3), Devmajor: -1},
[]byte("foo"),
}, {
&Header{Name: "world", Mode: 0640, Size: int64(5), Devmajor: -1},
[]byte("hello"),
}},
}, {
"PAX",
[]file{{
&Header{Name: "bar", Mode: 0640, Size: int64(3), Xattrs: map[string]string{"foo": "bar"}},
[]byte("foo"),
}, {
&Header{Name: "world", Mode: 0640, Size: int64(5), Xattrs: map[string]string{"foo": "bar"}},
[]byte("hello"),
}},
}}
b.Run("Writer", func(b *testing.B) {
for _, v := range vectors {
b.Run(v.label, func(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
// Writing to ioutil.Discard because we want to
// test purely the writer code and not bring in disk performance into this.
tw := NewWriter(ioutil.Discard)
for _, file := range v.files {
if err := tw.WriteHeader(file.hdr); err != nil {
b.Errorf("unexpected WriteHeader error: %v", err)
}
if _, err := tw.Write(file.body); err != nil {
b.Errorf("unexpected Write error: %v", err)
}
}
if err := tw.Close(); err != nil {
b.Errorf("unexpected Close error: %v", err)
}
}
})
}
})
b.Run("Reader", func(b *testing.B) {
for _, v := range vectors {
var buf bytes.Buffer
var r bytes.Reader
// Write the archive to a byte buffer.
tw := NewWriter(&buf)
for _, file := range v.files {
tw.WriteHeader(file.hdr)
tw.Write(file.body)
}
tw.Close()
b.Run(v.label, func(b *testing.B) {
b.ReportAllocs()
// Read from the byte buffer.
for i := 0; i < b.N; i++ {
r.Reset(buf.Bytes())
tr := NewReader(&r)
if _, err := tr.Next(); err != nil {
b.Errorf("unexpected Next error: %v", err)
}
if _, err := io.Copy(ioutil.Discard, tr); err != nil {
b.Errorf("unexpected Copy error : %v", err)
}
}
})
}
})
}

BIN
libgo/go/archive/tar/testdata/gnu-long-nul.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/gnu-nil-sparse-data.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/gnu-nil-sparse-hole.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/gnu-not-utf8.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/gnu-sparse-big.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/gnu-utf8.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/invalid-go17.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/pax-global-records.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/pax-nil-sparse-data.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/ustar.issue12594.tar → libgo/go/archive/tar/testdata/pax-nil-sparse-hole.tar vendored
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/pax-nul-path.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/pax-nul-xattrs.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/pax-pos-size-file.tar vendored
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/pax-records.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/pax-sparse-big.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/trailing-slash.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/ustar-file-devs.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/writer-big-long.tar vendored
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/writer-big.tar vendored
View file

Binary file not shown.

868
libgo/go/archive/tar/writer.go
View file

@ -4,255 +4,391 @@
package tar
// TODO(dsymonds):
// - catch more errors (no first header, etc.)
import (
"bytes"
"errors"
"fmt"
"io"
"path"
"sort"
"strconv"
"strings"
"time"
)
var (
ErrWriteTooLong = errors.New("archive/tar: write too long")
ErrFieldTooLong = errors.New("archive/tar: header field too long")
ErrWriteAfterClose = errors.New("archive/tar: write after close")
errInvalidHeader = errors.New("archive/tar: header field too long or contains invalid values")
)
// A Writer provides sequential writing of a tar archive in POSIX.1 format.
// A tar archive consists of a sequence of files.
// Call WriteHeader to begin a new file, and then call Write to supply that file's data,
// writing at most hdr.Size bytes in total.
// Writer provides sequential writing of a tar archive.
// Write.WriteHeader begins a new file with the provided Header,
// and then Writer can be treated as an io.Writer to supply that file's data.
type Writer struct {
w io.Writer
err error
nb int64 // number of unwritten bytes for current file entry
pad int64 // amount of padding to write after current file entry
closed bool
usedBinary bool // whether the binary numeric field extension was used
preferPax bool // use PAX header instead of binary numeric header
hdrBuff block // buffer to use in writeHeader when writing a regular header
paxHdrBuff block // buffer to use in writeHeader when writing a PAX header
w io.Writer
pad int64 // Amount of padding to write after current file entry
curr fileWriter // Writer for current file entry
hdr Header // Shallow copy of Header that is safe for mutations
blk block // Buffer to use as temporary local storage
// err is a persistent error.
// It is only the responsibility of every exported method of Writer to
// ensure that this error is sticky.
err error
}
// NewWriter creates a new Writer writing to w.
func NewWriter(w io.Writer) *Writer { return &Writer{w: w} }
func NewWriter(w io.Writer) *Writer {
return &Writer{w: w, curr: &regFileWriter{w, 0}}
}
// Flush finishes writing the current file (optional).
type fileWriter interface {
io.Writer
fileState
ReadFrom(io.Reader) (int64, error)
}
// Flush finishes writing the current file's block padding.
// The current file must be fully written before Flush can be called.
//
// This is unnecessary as the next call to WriteHeader or Close
// will implicitly flush out the file's padding.
func (tw *Writer) Flush() error {
if tw.nb > 0 {
tw.err = fmt.Errorf("archive/tar: missed writing %d bytes", tw.nb)
return tw.err
}
n := tw.nb + tw.pad
for n > 0 && tw.err == nil {
nr := n
if nr > blockSize {
nr = blockSize
}
var nw int
nw, tw.err = tw.w.Write(zeroBlock[0:nr])
n -= int64(nw)
}
tw.nb = 0
tw.pad = 0
return tw.err
}
var (
minTime = time.Unix(0, 0)
// There is room for 11 octal digits (33 bits) of mtime.
maxTime = minTime.Add((1<<33 - 1) * time.Second)
)
// WriteHeader writes hdr and prepares to accept the file's contents.
// WriteHeader calls Flush if it is not the first header.
// Calling after a Close will return ErrWriteAfterClose.
func (tw *Writer) WriteHeader(hdr *Header) error {
return tw.writeHeader(hdr, true)
}
// WriteHeader writes hdr and prepares to accept the file's contents.
// WriteHeader calls Flush if it is not the first header.
// Calling after a Close will return ErrWriteAfterClose.
// As this method is called internally by writePax header to allow it to
// suppress writing the pax header.
func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
if tw.closed {
return ErrWriteAfterClose
}
if tw.err == nil {
tw.Flush()
}
if tw.err != nil {
return tw.err
}
// a map to hold pax header records, if any are needed
paxHeaders := make(map[string]string)
// TODO(dsnet): we might want to use PAX headers for
// subsecond time resolution, but for now let's just capture
// too long fields or non ascii characters
// We need to select which scratch buffer to use carefully,
// since this method is called recursively to write PAX headers.
// If allowPax is true, this is the non-recursive call, and we will use hdrBuff.
// If allowPax is false, we are being called by writePAXHeader, and hdrBuff is
// already being used by the non-recursive call, so we must use paxHdrBuff.
header := &tw.hdrBuff
if !allowPax {
header = &tw.paxHdrBuff
if nb := tw.curr.LogicalRemaining(); nb > 0 {
return fmt.Errorf("archive/tar: missed writing %d bytes", nb)
}
copy(header[:], zeroBlock[:])
// Wrappers around formatter that automatically sets paxHeaders if the
// argument extends beyond the capacity of the input byte slice.
var f formatter
var formatString = func(b []byte, s string, paxKeyword string) {
needsPaxHeader := paxKeyword != paxNone && len(s) > len(b) || !isASCII(s)
if needsPaxHeader {
paxHeaders[paxKeyword] = s
}
// Write string in a best-effort manner to satisfy readers that expect
// the field to be non-empty.
s = toASCII(s)
if len(s) > len(b) {
s = s[:len(b)]
}
f.formatString(b, s) // Should never error
}
var formatNumeric = func(b []byte, x int64, paxKeyword string) {
// Try octal first.
s := strconv.FormatInt(x, 8)
if len(s) < len(b) {
f.formatOctal(b, x)
return
}
// If it is too long for octal, and PAX is preferred, use a PAX header.
if paxKeyword != paxNone && tw.preferPax {
f.formatOctal(b, 0)
s := strconv.FormatInt(x, 10)
paxHeaders[paxKeyword] = s
return
}
tw.usedBinary = true
f.formatNumeric(b, x)
}
// Handle out of range ModTime carefully.
var modTime int64
if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) {
modTime = hdr.ModTime.Unix()
}
v7 := header.V7()
formatString(v7.Name(), hdr.Name, paxPath)
// TODO(dsnet): The GNU format permits the mode field to be encoded in
// base-256 format. Thus, we can use formatNumeric instead of formatOctal.
f.formatOctal(v7.Mode(), hdr.Mode)
formatNumeric(v7.UID(), int64(hdr.Uid), paxUid)
formatNumeric(v7.GID(), int64(hdr.Gid), paxGid)
formatNumeric(v7.Size(), hdr.Size, paxSize)
// TODO(dsnet): Consider using PAX for finer time granularity.
formatNumeric(v7.ModTime(), modTime, paxNone)
v7.TypeFlag()[0] = hdr.Typeflag
formatString(v7.LinkName(), hdr.Linkname, paxLinkpath)
ustar := header.USTAR()
formatString(ustar.UserName(), hdr.Uname, paxUname)
formatString(ustar.GroupName(), hdr.Gname, paxGname)
formatNumeric(ustar.DevMajor(), hdr.Devmajor, paxNone)
formatNumeric(ustar.DevMinor(), hdr.Devminor, paxNone)
// TODO(dsnet): The logic surrounding the prefix field is broken when trying
// to encode the header as GNU format. The challenge with the current logic
// is that we are unsure what format we are using at any given moment until
// we have processed *all* of the fields. The problem is that by the time
// all fields have been processed, some work has already been done to handle
// each field under the assumption that it is for one given format or
// another. In some situations, this causes the Writer to be confused and
// encode a prefix field when the format being used is GNU. Thus, producing
// an invalid tar file.
//
// As a short-term fix, we disable the logic to use the prefix field, which
// will force the badly generated GNU files to become encoded as being
// the PAX format.
//
// As an alternative fix, we could hard-code preferPax to be true. However,
// this is problematic for the following reasons:
// * The preferPax functionality is not tested at all.
// * This can result in headers that try to use both the GNU and PAX
// features at the same time, which is also wrong.
//
// The proper fix for this is to use a two-pass method:
// * The first pass simply determines what set of formats can possibly
// encode the given header.
// * The second pass actually encodes the header as that given format
// without worrying about violating the format.
//
// See the following:
// https://golang.org/issue/12594
// https://golang.org/issue/17630
// https://golang.org/issue/9683
const usePrefix = false
// try to use a ustar header when only the name is too long
_, paxPathUsed := paxHeaders[paxPath]
if usePrefix && !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed {
prefix, suffix, ok := splitUSTARPath(hdr.Name)
if ok {
// Since we can encode in USTAR format, disable PAX header.
delete(paxHeaders, paxPath)
// Update the path fields
formatString(v7.Name(), suffix, paxNone)
formatString(ustar.Prefix(), prefix, paxNone)
}
}
if tw.usedBinary {
header.SetFormat(formatGNU)
} else {
header.SetFormat(formatUSTAR)
}
// Check if there were any formatting errors.
if f.err != nil {
tw.err = f.err
if _, tw.err = tw.w.Write(zeroBlock[:tw.pad]); tw.err != nil {
return tw.err
}
tw.pad = 0
return nil
}
if allowPax {
for k, v := range hdr.Xattrs {
paxHeaders[paxXattr+k] = v
// WriteHeader writes hdr and prepares to accept the file's contents.
// The Header.Size determines how many bytes can be written for the next file.
// If the current file is not fully written, then this returns an error.
// This implicitly flushes any padding necessary before writing the header.
func (tw *Writer) WriteHeader(hdr *Header) error {
if err := tw.Flush(); err != nil {
return err
}
tw.hdr = *hdr // Shallow copy of Header
// Round ModTime and ignore AccessTime and ChangeTime unless
// the format is explicitly chosen.
// This ensures nominal usage of WriteHeader (without specifying the format)
// does not always result in the PAX format being chosen, which
// causes a 1KiB increase to every header.
if tw.hdr.Format == FormatUnknown {
tw.hdr.ModTime = tw.hdr.ModTime.Round(time.Second)
tw.hdr.AccessTime = time.Time{}
tw.hdr.ChangeTime = time.Time{}
}
allowedFormats, paxHdrs, err := tw.hdr.allowedFormats()
switch {
case allowedFormats.has(FormatUSTAR):
tw.err = tw.writeUSTARHeader(&tw.hdr)
return tw.err
case allowedFormats.has(FormatPAX):
tw.err = tw.writePAXHeader(&tw.hdr, paxHdrs)
return tw.err
case allowedFormats.has(FormatGNU):
tw.err = tw.writeGNUHeader(&tw.hdr)
return tw.err
default:
return err // Non-fatal error
}
}
func (tw *Writer) writeUSTARHeader(hdr *Header) error {
// Check if we can use USTAR prefix/suffix splitting.
var namePrefix string
if prefix, suffix, ok := splitUSTARPath(hdr.Name); ok {
namePrefix, hdr.Name = prefix, suffix
}
// Pack the main header.
var f formatter
blk := tw.templateV7Plus(hdr, f.formatString, f.formatOctal)
f.formatString(blk.USTAR().Prefix(), namePrefix)
blk.SetFormat(FormatUSTAR)
if f.err != nil {
return f.err // Should never happen since header is validated
}
return tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag)
}
func (tw *Writer) writePAXHeader(hdr *Header, paxHdrs map[string]string) error {
realName, realSize := hdr.Name, hdr.Size
// TODO(dsnet): Re-enable this when adding sparse support.
// See https://golang.org/issue/22735
/*
// Handle sparse files.
var spd sparseDatas
var spb []byte
if len(hdr.SparseHoles) > 0 {
sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map
sph = alignSparseEntries(sph, hdr.Size)
spd = invertSparseEntries(sph, hdr.Size)
// Format the sparse map.
hdr.Size = 0 // Replace with encoded size
spb = append(strconv.AppendInt(spb, int64(len(spd)), 10), '\n')
for _, s := range spd {
hdr.Size += s.Length
spb = append(strconv.AppendInt(spb, s.Offset, 10), '\n')
spb = append(strconv.AppendInt(spb, s.Length, 10), '\n')
}
pad := blockPadding(int64(len(spb)))
spb = append(spb, zeroBlock[:pad]...)
hdr.Size += int64(len(spb)) // Accounts for encoded sparse map
// Add and modify appropriate PAX records.
dir, file := path.Split(realName)
hdr.Name = path.Join(dir, "GNUSparseFile.0", file)
paxHdrs[paxGNUSparseMajor] = "1"
paxHdrs[paxGNUSparseMinor] = "0"
paxHdrs[paxGNUSparseName] = realName
paxHdrs[paxGNUSparseRealSize] = strconv.FormatInt(realSize, 10)
paxHdrs[paxSize] = strconv.FormatInt(hdr.Size, 10)
delete(paxHdrs, paxPath) // Recorded by paxGNUSparseName
}
*/
_ = realSize
// Write PAX records to the output.
isGlobal := hdr.Typeflag == TypeXGlobalHeader
if len(paxHdrs) > 0 || isGlobal {
// Sort keys for deterministic ordering.
var keys []string
for k := range paxHdrs {
keys = append(keys, k)
}
sort.Strings(keys)
// Write each record to a buffer.
var buf bytes.Buffer
for _, k := range keys {
rec, err := formatPAXRecord(k, paxHdrs[k])
if err != nil {
return err
}
buf.WriteString(rec)
}
// Write the extended header file.
var name string
var flag byte
if isGlobal {
name = realName
if name == "" {
name = "GlobalHead.0.0"
}
flag = TypeXGlobalHeader
} else {
dir, file := path.Split(realName)
name = path.Join(dir, "PaxHeaders.0", file)
flag = TypeXHeader
}
data := buf.String()
if err := tw.writeRawFile(name, data, flag, FormatPAX); err != nil || isGlobal {
return err // Global headers return here
}
}
if len(paxHeaders) > 0 {
if !allowPax {
return errInvalidHeader
// Pack the main header.
var f formatter // Ignore errors since they are expected
fmtStr := func(b []byte, s string) { f.formatString(b, toASCII(s)) }
blk := tw.templateV7Plus(hdr, fmtStr, f.formatOctal)
blk.SetFormat(FormatPAX)
if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil {
return err
}
// TODO(dsnet): Re-enable this when adding sparse support.
// See https://golang.org/issue/22735
/*
// Write the sparse map and setup the sparse writer if necessary.
if len(spd) > 0 {
// Use tw.curr since the sparse map is accounted for in hdr.Size.
if _, err := tw.curr.Write(spb); err != nil {
return err
}
tw.curr = &sparseFileWriter{tw.curr, spd, 0}
}
if err := tw.writePAXHeader(hdr, paxHeaders); err != nil {
*/
return nil
}
func (tw *Writer) writeGNUHeader(hdr *Header) error {
// Use long-link files if Name or Linkname exceeds the field size.
const longName = "././@LongLink"
if len(hdr.Name) > nameSize {
data := hdr.Name + "\x00"
if err := tw.writeRawFile(longName, data, TypeGNULongName, FormatGNU); err != nil {
return err
}
}
if len(hdr.Linkname) > nameSize {
data := hdr.Linkname + "\x00"
if err := tw.writeRawFile(longName, data, TypeGNULongLink, FormatGNU); err != nil {
return err
}
}
tw.nb = hdr.Size
tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize
_, tw.err = tw.w.Write(header[:])
return tw.err
// Pack the main header.
var f formatter // Ignore errors since they are expected
var spd sparseDatas
var spb []byte
blk := tw.templateV7Plus(hdr, f.formatString, f.formatNumeric)
if !hdr.AccessTime.IsZero() {
f.formatNumeric(blk.GNU().AccessTime(), hdr.AccessTime.Unix())
}
if !hdr.ChangeTime.IsZero() {
f.formatNumeric(blk.GNU().ChangeTime(), hdr.ChangeTime.Unix())
}
// TODO(dsnet): Re-enable this when adding sparse support.
// See https://golang.org/issue/22735
/*
if hdr.Typeflag == TypeGNUSparse {
sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map
sph = alignSparseEntries(sph, hdr.Size)
spd = invertSparseEntries(sph, hdr.Size)
// Format the sparse map.
formatSPD := func(sp sparseDatas, sa sparseArray) sparseDatas {
for i := 0; len(sp) > 0 && i < sa.MaxEntries(); i++ {
f.formatNumeric(sa.Entry(i).Offset(), sp[0].Offset)
f.formatNumeric(sa.Entry(i).Length(), sp[0].Length)
sp = sp[1:]
}
if len(sp) > 0 {
sa.IsExtended()[0] = 1
}
return sp
}
sp2 := formatSPD(spd, blk.GNU().Sparse())
for len(sp2) > 0 {
var spHdr block
sp2 = formatSPD(sp2, spHdr.Sparse())
spb = append(spb, spHdr[:]...)
}
// Update size fields in the header block.
realSize := hdr.Size
hdr.Size = 0 // Encoded size; does not account for encoded sparse map
for _, s := range spd {
hdr.Size += s.Length
}
copy(blk.V7().Size(), zeroBlock[:]) // Reset field
f.formatNumeric(blk.V7().Size(), hdr.Size)
f.formatNumeric(blk.GNU().RealSize(), realSize)
}
*/
blk.SetFormat(FormatGNU)
if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil {
return err
}
// Write the extended sparse map and setup the sparse writer if necessary.
if len(spd) > 0 {
// Use tw.w since the sparse map is not accounted for in hdr.Size.
if _, err := tw.w.Write(spb); err != nil {
return err
}
tw.curr = &sparseFileWriter{tw.curr, spd, 0}
}
return nil
}
type (
stringFormatter func([]byte, string)
numberFormatter func([]byte, int64)
)
// templateV7Plus fills out the V7 fields of a block using values from hdr.
// It also fills out fields (uname, gname, devmajor, devminor) that are
// shared in the USTAR, PAX, and GNU formats using the provided formatters.
//
// The block returned is only valid until the next call to
// templateV7Plus or writeRawFile.
func (tw *Writer) templateV7Plus(hdr *Header, fmtStr stringFormatter, fmtNum numberFormatter) *block {
tw.blk.Reset()
modTime := hdr.ModTime
if modTime.IsZero() {
modTime = time.Unix(0, 0)
}
v7 := tw.blk.V7()
v7.TypeFlag()[0] = hdr.Typeflag
fmtStr(v7.Name(), hdr.Name)
fmtStr(v7.LinkName(), hdr.Linkname)
fmtNum(v7.Mode(), hdr.Mode)
fmtNum(v7.UID(), int64(hdr.Uid))
fmtNum(v7.GID(), int64(hdr.Gid))
fmtNum(v7.Size(), hdr.Size)
fmtNum(v7.ModTime(), modTime.Unix())
ustar := tw.blk.USTAR()
fmtStr(ustar.UserName(), hdr.Uname)
fmtStr(ustar.GroupName(), hdr.Gname)
fmtNum(ustar.DevMajor(), hdr.Devmajor)
fmtNum(ustar.DevMinor(), hdr.Devminor)
return &tw.blk
}
// writeRawFile writes a minimal file with the given name and flag type.
// It uses format to encode the header format and will write data as the body.
// It uses default values for all of the other fields (as BSD and GNU tar does).
func (tw *Writer) writeRawFile(name, data string, flag byte, format Format) error {
tw.blk.Reset()
// Best effort for the filename.
name = toASCII(name)
if len(name) > nameSize {
name = name[:nameSize]
}
name = strings.TrimRight(name, "/")
var f formatter
v7 := tw.blk.V7()
v7.TypeFlag()[0] = flag
f.formatString(v7.Name(), name)
f.formatOctal(v7.Mode(), 0)
f.formatOctal(v7.UID(), 0)
f.formatOctal(v7.GID(), 0)
f.formatOctal(v7.Size(), int64(len(data))) // Must be < 8GiB
f.formatOctal(v7.ModTime(), 0)
tw.blk.SetFormat(format)
if f.err != nil {
return f.err // Only occurs if size condition is violated
}
// Write the header and data.
if err := tw.writeRawHeader(&tw.blk, int64(len(data)), flag); err != nil {
return err
}
_, err := io.WriteString(tw, data)
return err
}
// writeRawHeader writes the value of blk, regardless of its value.
// It sets up the Writer such that it can accept a file of the given size.
// If the flag is a special header-only flag, then the size is treated as zero.
func (tw *Writer) writeRawHeader(blk *block, size int64, flag byte) error {
if err := tw.Flush(); err != nil {
return err
}
if _, err := tw.w.Write(blk[:]); err != nil {
return err
}
if isHeaderOnlyType(flag) {
size = 0
}
tw.curr = &regFileWriter{tw.w, size}
tw.pad = blockPadding(size)
return nil
}
// splitUSTARPath splits a path according to USTAR prefix and suffix rules.
@ -276,95 +412,233 @@ func splitUSTARPath(name string) (prefix, suffix string, ok bool) {
return name[:i], name[i+1:], true
}
// writePaxHeader writes an extended pax header to the
// archive.
func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error {
// Prepare extended header
ext := new(Header)
ext.Typeflag = TypeXHeader
// Setting ModTime is required for reader parsing to
// succeed, and seems harmless enough.
ext.ModTime = hdr.ModTime
// The spec asks that we namespace our pseudo files
// with the current pid. However, this results in differing outputs
// for identical inputs. As such, the constant 0 is now used instead.
// golang.org/issue/12358
dir, file := path.Split(hdr.Name)
fullName := path.Join(dir, "PaxHeaders.0", file)
ascii := toASCII(fullName)
if len(ascii) > nameSize {
ascii = ascii[:nameSize]
}
ext.Name = ascii
// Construct the body
var buf bytes.Buffer
// Keys are sorted before writing to body to allow deterministic output.
keys := make([]string, 0, len(paxHeaders))
for k := range paxHeaders {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
fmt.Fprint(&buf, formatPAXRecord(k, paxHeaders[k]))
}
ext.Size = int64(len(buf.Bytes()))
if err := tw.writeHeader(ext, false); err != nil {
return err
}
if _, err := tw.Write(buf.Bytes()); err != nil {
return err
}
if err := tw.Flush(); err != nil {
return err
}
return nil
}
// Write writes to the current entry in the tar archive.
// Write writes to the current file in the tar archive.
// Write returns the error ErrWriteTooLong if more than
// hdr.Size bytes are written after WriteHeader.
func (tw *Writer) Write(b []byte) (n int, err error) {
if tw.closed {
err = ErrWriteAfterClose
return
// Header.Size bytes are written after WriteHeader.
//
// Calling Write on special types like TypeLink, TypeSymlink, TypeChar,
// TypeBlock, TypeDir, and TypeFifo returns (0, ErrWriteTooLong) regardless
// of what the Header.Size claims.
func (tw *Writer) Write(b []byte) (int, error) {
if tw.err != nil {
return 0, tw.err
}
overwrite := false
if int64(len(b)) > tw.nb {
b = b[0:tw.nb]
overwrite = true
n, err := tw.curr.Write(b)
if err != nil && err != ErrWriteTooLong {
tw.err = err
}
n, err = tw.w.Write(b)
tw.nb -= int64(n)
if err == nil && overwrite {
err = ErrWriteTooLong
return
}
tw.err = err
return
return n, err
}
// Close closes the tar archive, flushing any unwritten
// data to the underlying writer.
func (tw *Writer) Close() error {
if tw.err != nil || tw.closed {
return tw.err
// readFrom populates the content of the current file by reading from r.
// The bytes read must match the number of remaining bytes in the current file.
//
// If the current file is sparse and r is an io.ReadSeeker,
// then readFrom uses Seek to skip past holes defined in Header.SparseHoles,
// assuming that skipped regions are all NULs.
// This always reads the last byte to ensure r is the right size.
//
// TODO(dsnet): Re-export this when adding sparse file support.
// See https://golang.org/issue/22735
func (tw *Writer) readFrom(r io.Reader) (int64, error) {
if tw.err != nil {
return 0, tw.err
}
n, err := tw.curr.ReadFrom(r)
if err != nil && err != ErrWriteTooLong {
tw.err = err
}
return n, err
}
// Close closes the tar archive by flushing the padding, and writing the footer.
// If the current file (from a prior call to WriteHeader) is not fully written,
// then this returns an error.
func (tw *Writer) Close() error {
if tw.err == ErrWriteAfterClose {
return nil
}
tw.Flush()
tw.closed = true
if tw.err != nil {
return tw.err
}
// trailer: two zero blocks
for i := 0; i < 2; i++ {
_, tw.err = tw.w.Write(zeroBlock[:])
if tw.err != nil {
break
// Trailer: two zero blocks.
err := tw.Flush()
for i := 0; i < 2 && err == nil; i++ {
_, err = tw.w.Write(zeroBlock[:])
}
// Ensure all future actions are invalid.
tw.err = ErrWriteAfterClose
return err // Report IO errors
}
// regFileWriter is a fileWriter for writing data to a regular file entry.
type regFileWriter struct {
w io.Writer // Underlying Writer
nb int64 // Number of remaining bytes to write
}
func (fw *regFileWriter) Write(b []byte) (n int, err error) {
overwrite := int64(len(b)) > fw.nb
if overwrite {
b = b[:fw.nb]
}
if len(b) > 0 {
n, err = fw.w.Write(b)
fw.nb -= int64(n)
}
switch {
case err != nil:
return n, err
case overwrite:
return n, ErrWriteTooLong
default:
return n, nil
}
}
func (fw *regFileWriter) ReadFrom(r io.Reader) (int64, error) {
return io.Copy(struct{ io.Writer }{fw}, r)
}
func (fw regFileWriter) LogicalRemaining() int64 {
return fw.nb
}
func (fw regFileWriter) PhysicalRemaining() int64 {
return fw.nb
}
// sparseFileWriter is a fileWriter for writing data to a sparse file entry.
type sparseFileWriter struct {
fw fileWriter // Underlying fileWriter
sp sparseDatas // Normalized list of data fragments
pos int64 // Current position in sparse file
}
func (sw *sparseFileWriter) Write(b []byte) (n int, err error) {
overwrite := int64(len(b)) > sw.LogicalRemaining()
if overwrite {
b = b[:sw.LogicalRemaining()]
}
b0 := b
endPos := sw.pos + int64(len(b))
for endPos > sw.pos && err == nil {
var nf int // Bytes written in fragment
dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset()
if sw.pos < dataStart { // In a hole fragment
bf := b[:min(int64(len(b)), dataStart-sw.pos)]
nf, err = zeroWriter{}.Write(bf)
} else { // In a data fragment
bf := b[:min(int64(len(b)), dataEnd-sw.pos)]
nf, err = sw.fw.Write(bf)
}
b = b[nf:]
sw.pos += int64(nf)
if sw.pos >= dataEnd && len(sw.sp) > 1 {
sw.sp = sw.sp[1:] // Ensure last fragment always remains
}
}
return tw.err
n = len(b0) - len(b)
switch {
case err == ErrWriteTooLong:
return n, errMissData // Not possible; implies bug in validation logic
case err != nil:
return n, err
case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0:
return n, errUnrefData // Not possible; implies bug in validation logic
case overwrite:
return n, ErrWriteTooLong
default:
return n, nil
}
}
func (sw *sparseFileWriter) ReadFrom(r io.Reader) (n int64, err error) {
rs, ok := r.(io.ReadSeeker)
if ok {
if _, err := rs.Seek(0, io.SeekCurrent); err != nil {
ok = false // Not all io.Seeker can really seek
}
}
if !ok {
return io.Copy(struct{ io.Writer }{sw}, r)
}
var readLastByte bool
pos0 := sw.pos
for sw.LogicalRemaining() > 0 && !readLastByte && err == nil {
var nf int64 // Size of fragment
dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset()
if sw.pos < dataStart { // In a hole fragment
nf = dataStart - sw.pos
if sw.PhysicalRemaining() == 0 {
readLastByte = true
nf--
}
_, err = rs.Seek(nf, io.SeekCurrent)
} else { // In a data fragment
nf = dataEnd - sw.pos
nf, err = io.CopyN(sw.fw, rs, nf)
}
sw.pos += nf
if sw.pos >= dataEnd && len(sw.sp) > 1 {
sw.sp = sw.sp[1:] // Ensure last fragment always remains
}
}
// If the last fragment is a hole, then seek to 1-byte before EOF, and
// read a single byte to ensure the file is the right size.
if readLastByte && err == nil {
_, err = mustReadFull(rs, []byte{0})
sw.pos++
}
n = sw.pos - pos0
switch {
case err == io.EOF:
return n, io.ErrUnexpectedEOF
case err == ErrWriteTooLong:
return n, errMissData // Not possible; implies bug in validation logic
case err != nil:
return n, err
case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0:
return n, errUnrefData // Not possible; implies bug in validation logic
default:
return n, ensureEOF(rs)
}
}
func (sw sparseFileWriter) LogicalRemaining() int64 {
return sw.sp[len(sw.sp)-1].endOffset() - sw.pos
}
func (sw sparseFileWriter) PhysicalRemaining() int64 {
return sw.fw.PhysicalRemaining()
}
// zeroWriter may only be written with NULs, otherwise it returns errWriteHole.
type zeroWriter struct{}
func (zeroWriter) Write(b []byte) (int, error) {
for i, c := range b {
if c != 0 {
return i, errWriteHole
}
}
return len(b), nil
}
// ensureEOF checks whether r is at EOF, reporting ErrWriteTooLong if not so.
func ensureEOF(r io.Reader) error {
n, err := tryReadFull(r, []byte{0})
switch {
case n > 0:
return ErrWriteTooLong
case err == io.EOF:
return nil
default:
return err
}
}

1007
libgo/go/archive/tar/writer_test.go
View file

File diff suppressed because it is too large Load diff

169
libgo/go/archive/zip/reader.go
View file

@ -13,6 +13,7 @@ import (
"hash/crc32"
"io"
"os"
"time"
)
var (
@ -94,7 +95,7 @@ func (z *Reader) init(r io.ReaderAt, size int64) error {
// The count of files inside a zip is truncated to fit in a uint16.
// Gloss over this by reading headers until we encounter
// a bad one, and then only report a ErrFormat or UnexpectedEOF if
// a bad one, and then only report an ErrFormat or UnexpectedEOF if
// the file count modulo 65536 is incorrect.
for {
f := &File{zip: z, zipr: r, zipsize: size}
@ -280,52 +281,128 @@ func readDirectoryHeader(f *File, r io.Reader) error {
f.Extra = d[filenameLen : filenameLen+extraLen]
f.Comment = string(d[filenameLen+extraLen:])
// Determine the character encoding.
utf8Valid1, utf8Require1 := detectUTF8(f.Name)
utf8Valid2, utf8Require2 := detectUTF8(f.Comment)
switch {
case !utf8Valid1 || !utf8Valid2:
// Name and Comment definitely not UTF-8.
f.NonUTF8 = true
case !utf8Require1 && !utf8Require2:
// Name and Comment use only single-byte runes that overlap with UTF-8.
f.NonUTF8 = false
default:
// Might be UTF-8, might be some other encoding; preserve existing flag.
// Some ZIP writers use UTF-8 encoding without setting the UTF-8 flag.
// Since it is impossible to always distinguish valid UTF-8 from some
// other encoding (e.g., GBK or Shift-JIS), we trust the flag.
f.NonUTF8 = f.Flags&0x800 == 0
}
needUSize := f.UncompressedSize == ^uint32(0)
needCSize := f.CompressedSize == ^uint32(0)
needHeaderOffset := f.headerOffset == int64(^uint32(0))
if len(f.Extra) > 0 {
// Best effort to find what we need.
// Other zip authors might not even follow the basic format,
// and we'll just ignore the Extra content in that case.
b := readBuf(f.Extra)
for len(b) >= 4 { // need at least tag and size
tag := b.uint16()
size := b.uint16()
if int(size) > len(b) {
break
}
if tag == zip64ExtraId {
// update directory values from the zip64 extra block.
// They should only be consulted if the sizes read earlier
// are maxed out.
// See golang.org/issue/13367.
eb := readBuf(b[:size])
// Best effort to find what we need.
// Other zip authors might not even follow the basic format,
// and we'll just ignore the Extra content in that case.
var modified time.Time
parseExtras:
for extra := readBuf(f.Extra); len(extra) >= 4; { // need at least tag and size
fieldTag := extra.uint16()
fieldSize := int(extra.uint16())
if len(extra) < fieldSize {
break
}
fieldBuf := extra.sub(fieldSize)
if needUSize {
needUSize = false
if len(eb) < 8 {
return ErrFormat
}
f.UncompressedSize64 = eb.uint64()
switch fieldTag {
case zip64ExtraID:
// update directory values from the zip64 extra block.
// They should only be consulted if the sizes read earlier
// are maxed out.
// See golang.org/issue/13367.
if needUSize {
needUSize = false
if len(fieldBuf) < 8 {
return ErrFormat
}
if needCSize {
needCSize = false
if len(eb) < 8 {
return ErrFormat
}
f.CompressedSize64 = eb.uint64()
}
if needHeaderOffset {
needHeaderOffset = false
if len(eb) < 8 {
return ErrFormat
}
f.headerOffset = int64(eb.uint64())
}
break
f.UncompressedSize64 = fieldBuf.uint64()
}
b = b[size:]
if needCSize {
needCSize = false
if len(fieldBuf) < 8 {
return ErrFormat
}
f.CompressedSize64 = fieldBuf.uint64()
}
if needHeaderOffset {
needHeaderOffset = false
if len(fieldBuf) < 8 {
return ErrFormat
}
f.headerOffset = int64(fieldBuf.uint64())
}
case ntfsExtraID:
if len(fieldBuf) < 4 {
continue parseExtras
}
fieldBuf.uint32() // reserved (ignored)
for len(fieldBuf) >= 4 { // need at least tag and size
attrTag := fieldBuf.uint16()
attrSize := int(fieldBuf.uint16())
if len(fieldBuf) < attrSize {
continue parseExtras
}
attrBuf := fieldBuf.sub(attrSize)
if attrTag != 1 || attrSize != 24 {
continue // Ignore irrelevant attributes
}
const ticksPerSecond = 1e7 // Windows timestamp resolution
ts := int64(attrBuf.uint64()) // ModTime since Windows epoch
secs := int64(ts / ticksPerSecond)
nsecs := (1e9 / ticksPerSecond) * int64(ts%ticksPerSecond)
epoch := time.Date(1601, time.January, 1, 0, 0, 0, 0, time.UTC)
modified = time.Unix(epoch.Unix()+secs, nsecs)
}
case unixExtraID:
if len(fieldBuf) < 8 {
continue parseExtras
}
fieldBuf.uint32() // AcTime (ignored)
ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
modified = time.Unix(ts, 0)
case extTimeExtraID:
if len(fieldBuf) < 5 || fieldBuf.uint8()&1 == 0 {
continue parseExtras
}
ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
modified = time.Unix(ts, 0)
case infoZipUnixExtraID:
if len(fieldBuf) < 4 {
continue parseExtras
}
ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
modified = time.Unix(ts, 0)
}
}
msdosModified := msDosTimeToTime(f.ModifiedDate, f.ModifiedTime)
f.Modified = msdosModified
if !modified.IsZero() {
f.Modified = modified.UTC()
// If legacy MS-DOS timestamps are set, we can use the delta between
// the legacy and extended versions to estimate timezone offset.
//
// A non-UTC timezone is always used (even if offset is zero).
// Thus, FileHeader.Modified.Location() == time.UTC is useful for
// determining whether extended timestamps are present.
// This is necessary for users that need to do additional time
// calculations when dealing with legacy ZIP formats.
if f.ModifiedTime != 0 || f.ModifiedDate != 0 {
f.Modified = modified.In(timeZone(msdosModified.Sub(modified)))
}
}
@ -508,6 +585,12 @@ func findSignatureInBlock(b []byte) int {
type readBuf []byte
func (b *readBuf) uint8() uint8 {
v := (*b)[0]
*b = (*b)[1:]
return v
}
func (b *readBuf) uint16() uint16 {
v := binary.LittleEndian.Uint16(*b)
*b = (*b)[2:]
@ -525,3 +608,9 @@ func (b *readBuf) uint64() uint64 {
*b = (*b)[8:]
return v
}
func (b *readBuf) sub(n int) readBuf {
b2 := (*b)[:n]
*b = (*b)[n:]
return b2
}

447
libgo/go/archive/zip/reader_test.go
View file

@ -27,9 +27,11 @@ type ZipTest struct {
}
type ZipTestFile struct {
Name string
Mode os.FileMode
Mtime string // optional, modified time in format "mm-dd-yy hh:mm:ss"
Name string
Mode os.FileMode
NonUTF8 bool
ModTime time.Time
Modified time.Time
// Information describing expected zip file content.
// First, reading the entire content should produce the error ContentErr.
@ -47,32 +49,22 @@ type ZipTestFile struct {
Size uint64
}
// Caution: The Mtime values found for the test files should correspond to
// the values listed with unzip -l <zipfile>. However, the values
// listed by unzip appear to be off by some hours. When creating
// fresh test files and testing them, this issue is not present.
// The test files were created in Sydney, so there might be a time
// zone issue. The time zone information does have to be encoded
// somewhere, because otherwise unzip -l could not provide a different
// time from what the archive/zip package provides, but there appears
// to be no documentation about this.
var tests = []ZipTest{
{
Name: "test.zip",
Comment: "This is a zipfile comment.",
File: []ZipTestFile{
{
Name: "test.txt",
Content: []byte("This is a test text file.\n"),
Mtime: "09-05-10 12:12:02",
Mode: 0644,
Name: "test.txt",
Content: []byte("This is a test text file.\n"),
Modified: time.Date(2010, 9, 5, 12, 12, 1, 0, timeZone(+10*time.Hour)),
Mode: 0644,
},
{
Name: "gophercolor16x16.png",
File: "gophercolor16x16.png",
Mtime: "09-05-10 15:52:58",
Mode: 0644,
Name: "gophercolor16x16.png",
File: "gophercolor16x16.png",
Modified: time.Date(2010, 9, 5, 15, 52, 58, 0, timeZone(+10*time.Hour)),
Mode: 0644,
},
},
},
@ -81,16 +73,16 @@ var tests = []ZipTest{
Comment: "This is a zipfile comment.",
File: []ZipTestFile{
{
Name: "test.txt",
Content: []byte("This is a test text file.\n"),
Mtime: "09-05-10 12:12:02",
Mode: 0644,
Name: "test.txt",
Content: []byte("This is a test text file.\n"),
Modified: time.Date(2010, 9, 5, 12, 12, 1, 0, timeZone(+10*time.Hour)),
Mode: 0644,
},
{
Name: "gophercolor16x16.png",
File: "gophercolor16x16.png",
Mtime: "09-05-10 15:52:58",
Mode: 0644,
Name: "gophercolor16x16.png",
File: "gophercolor16x16.png",
Modified: time.Date(2010, 9, 5, 15, 52, 58, 0, timeZone(+10*time.Hour)),
Mode: 0644,
},
},
},
@ -99,10 +91,10 @@ var tests = []ZipTest{
Source: returnRecursiveZip,
File: []ZipTestFile{
{
Name: "r/r.zip",
Content: rZipBytes(),
Mtime: "03-04-10 00:24:16",
Mode: 0666,
Name: "r/r.zip",
Content: rZipBytes(),
Modified: time.Date(2010, 3, 4, 0, 24, 16, 0, time.UTC),
Mode: 0666,
},
},
},
@ -110,9 +102,10 @@ var tests = []ZipTest{
Name: "symlink.zip",
File: []ZipTestFile{
{
Name: "symlink",
Content: []byte("../target"),
Mode: 0777 | os.ModeSymlink,
Name: "symlink",
Content: []byte("../target"),
Modified: time.Date(2012, 2, 3, 19, 56, 48, 0, timeZone(-2*time.Hour)),
Mode: 0777 | os.ModeSymlink,
},
},
},
@ -127,22 +120,72 @@ var tests = []ZipTest{
Name: "dd.zip",
File: []ZipTestFile{
{
Name: "filename",
Content: []byte("This is a test textfile.\n"),
Mtime: "02-02-11 13:06:20",
Mode: 0666,
Name: "filename",
Content: []byte("This is a test textfile.\n"),
Modified: time.Date(2011, 2, 2, 13, 6, 20, 0, time.UTC),
Mode: 0666,
},
},
},
{
// created in windows XP file manager.
Name: "winxp.zip",
File: crossPlatform,
File: []ZipTestFile{
{
Name: "hello",
Content: []byte("world \r\n"),
Modified: time.Date(2011, 12, 8, 10, 4, 24, 0, time.UTC),
Mode: 0666,
},
{
Name: "dir/bar",
Content: []byte("foo \r\n"),
Modified: time.Date(2011, 12, 8, 10, 4, 50, 0, time.UTC),
Mode: 0666,
},
{
Name: "dir/empty/",
Content: []byte{},
Modified: time.Date(2011, 12, 8, 10, 8, 6, 0, time.UTC),
Mode: os.ModeDir | 0777,
},
{
Name: "readonly",
Content: []byte("important \r\n"),
Modified: time.Date(2011, 12, 8, 10, 6, 8, 0, time.UTC),
Mode: 0444,
},
},
},
{
// created by Zip 3.0 under Linux
Name: "unix.zip",
File: crossPlatform,
File: []ZipTestFile{
{
Name: "hello",
Content: []byte("world \r\n"),
Modified: time.Date(2011, 12, 8, 10, 4, 24, 0, timeZone(0)),
Mode: 0666,
},
{
Name: "dir/bar",
Content: []byte("foo \r\n"),
Modified: time.Date(2011, 12, 8, 10, 4, 50, 0, timeZone(0)),
Mode: 0666,
},
{
Name: "dir/empty/",
Content: []byte{},
Modified: time.Date(2011, 12, 8, 10, 8, 6, 0, timeZone(0)),
Mode: os.ModeDir | 0777,
},
{
Name: "readonly",
Content: []byte("important \r\n"),
Modified: time.Date(2011, 12, 8, 10, 6, 8, 0, timeZone(0)),
Mode: 0444,
},
},
},
{
// created by Go, before we wrote the "optional" data
@ -150,16 +193,16 @@ var tests = []ZipTest{
Name: "go-no-datadesc-sig.zip",
File: []ZipTestFile{
{
Name: "foo.txt",
Content: []byte("foo\n"),
Mtime: "03-08-12 16:59:10",
Mode: 0644,
Name: "foo.txt",
Content: []byte("foo\n"),
Modified: time.Date(2012, 3, 8, 16, 59, 10, 0, timeZone(-8*time.Hour)),
Mode: 0644,
},
{
Name: "bar.txt",
Content: []byte("bar\n"),
Mtime: "03-08-12 16:59:12",
Mode: 0644,
Name: "bar.txt",
Content: []byte("bar\n"),
Modified: time.Date(2012, 3, 8, 16, 59, 12, 0, timeZone(-8*time.Hour)),
Mode: 0644,
},
},
},
@ -169,14 +212,16 @@ var tests = []ZipTest{
Name: "go-with-datadesc-sig.zip",
File: []ZipTestFile{
{
Name: "foo.txt",
Content: []byte("foo\n"),
Mode: 0666,
Name: "foo.txt",
Content: []byte("foo\n"),
Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
Mode: 0666,
},
{
Name: "bar.txt",
Content: []byte("bar\n"),
Mode: 0666,
Name: "bar.txt",
Content: []byte("bar\n"),
Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
Mode: 0666,
},
},
},
@ -187,13 +232,15 @@ var tests = []ZipTest{
{
Name: "foo.txt",
Content: []byte("foo\n"),
Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
Mode: 0666,
ContentErr: ErrChecksum,
},
{
Name: "bar.txt",
Content: []byte("bar\n"),
Mode: 0666,
Name: "bar.txt",
Content: []byte("bar\n"),
Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
Mode: 0666,
},
},
},
@ -203,16 +250,16 @@ var tests = []ZipTest{
Name: "crc32-not-streamed.zip",
File: []ZipTestFile{
{
Name: "foo.txt",
Content: []byte("foo\n"),
Mtime: "03-08-12 16:59:10",
Mode: 0644,
Name: "foo.txt",
Content: []byte("foo\n"),
Modified: time.Date(2012, 3, 8, 16, 59, 10, 0, timeZone(-8*time.Hour)),
Mode: 0644,
},
{
Name: "bar.txt",
Content: []byte("bar\n"),
Mtime: "03-08-12 16:59:12",
Mode: 0644,
Name: "bar.txt",
Content: []byte("bar\n"),
Modified: time.Date(2012, 3, 8, 16, 59, 12, 0, timeZone(-8*time.Hour)),
Mode: 0644,
},
},
},
@ -225,15 +272,15 @@ var tests = []ZipTest{
{
Name: "foo.txt",
Content: []byte("foo\n"),
Mtime: "03-08-12 16:59:10",
Modified: time.Date(2012, 3, 8, 16, 59, 10, 0, timeZone(-8*time.Hour)),
Mode: 0644,
ContentErr: ErrChecksum,
},
{
Name: "bar.txt",
Content: []byte("bar\n"),
Mtime: "03-08-12 16:59:12",
Mode: 0644,
Name: "bar.txt",
Content: []byte("bar\n"),
Modified: time.Date(2012, 3, 8, 16, 59, 12, 0, timeZone(-8*time.Hour)),
Mode: 0644,
},
},
},
@ -241,10 +288,10 @@ var tests = []ZipTest{
Name: "zip64.zip",
File: []ZipTestFile{
{
Name: "README",
Content: []byte("This small file is in ZIP64 format.\n"),
Mtime: "08-10-12 14:33:32",
Mode: 0644,
Name: "README",
Content: []byte("This small file is in ZIP64 format.\n"),
Modified: time.Date(2012, 8, 10, 14, 33, 32, 0, time.UTC),
Mode: 0644,
},
},
},
@ -253,10 +300,10 @@ var tests = []ZipTest{
Name: "zip64-2.zip",
File: []ZipTestFile{
{
Name: "README",
Content: []byte("This small file is in ZIP64 format.\n"),
Mtime: "08-10-12 14:33:32",
Mode: 0644,
Name: "README",
Content: []byte("This small file is in ZIP64 format.\n"),
Modified: time.Date(2012, 8, 10, 14, 33, 32, 0, timeZone(-4*time.Hour)),
Mode: 0644,
},
},
},
@ -266,41 +313,179 @@ var tests = []ZipTest{
Source: returnBigZipBytes,
File: []ZipTestFile{
{
Name: "big.file",
Content: nil,
Size: 1<<32 - 1,
Mode: 0666,
Name: "big.file",
Content: nil,
Size: 1<<32 - 1,
Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
Mode: 0666,
},
},
},
{
Name: "utf8-7zip.zip",
File: []ZipTestFile{
{
Name: "世界",
Content: []byte{},
Mode: 0666,
Modified: time.Date(2017, 11, 6, 13, 9, 27, 867862500, timeZone(-8*time.Hour)),
},
},
},
{
Name: "utf8-infozip.zip",
File: []ZipTestFile{
{
Name: "世界",
Content: []byte{},
Mode: 0644,
// Name is valid UTF-8, but format does not have UTF-8 flag set.
// We don't do UTF-8 detection for multi-byte runes due to
// false-positives with other encodings (e.g., Shift-JIS).
// Format says encoding is not UTF-8, so we trust it.
NonUTF8: true,
Modified: time.Date(2017, 11, 6, 13, 9, 27, 0, timeZone(-8*time.Hour)),
},
},
},
{
Name: "utf8-osx.zip",
File: []ZipTestFile{
{
Name: "世界",
Content: []byte{},
Mode: 0644,
// Name is valid UTF-8, but format does not have UTF-8 set.
NonUTF8: true,
Modified: time.Date(2017, 11, 6, 13, 9, 27, 0, timeZone(-8*time.Hour)),
},
},
},
{
Name: "utf8-winrar.zip",
File: []ZipTestFile{
{
Name: "世界",
Content: []byte{},
Mode: 0666,
Modified: time.Date(2017, 11, 6, 13, 9, 27, 867862500, timeZone(-8*time.Hour)),
},
},
},
{
Name: "utf8-winzip.zip",
File: []ZipTestFile{
{
Name: "世界",
Content: []byte{},
Mode: 0666,
Modified: time.Date(2017, 11, 6, 13, 9, 27, 867000000, timeZone(-8*time.Hour)),
},
},
},
{
Name: "time-7zip.zip",
File: []ZipTestFile{
{
Name: "test.txt",
Content: []byte{},
Size: 1<<32 - 1,
Modified: time.Date(2017, 10, 31, 21, 11, 57, 244817900, timeZone(-7*time.Hour)),
Mode: 0666,
},
},
},
{
Name: "time-infozip.zip",
File: []ZipTestFile{
{
Name: "test.txt",
Content: []byte{},
Size: 1<<32 - 1,
Modified: time.Date(2017, 10, 31, 21, 11, 57, 0, timeZone(-7*time.Hour)),
Mode: 0644,
},
},
},
{
Name: "time-osx.zip",
File: []ZipTestFile{
{
Name: "test.txt",
Content: []byte{},
Size: 1<<32 - 1,
Modified: time.Date(2017, 10, 31, 21, 17, 27, 0, timeZone(-7*time.Hour)),
Mode: 0644,
},
},
},
{
Name: "time-win7.zip",
File: []ZipTestFile{
{
Name: "test.txt",
Content: []byte{},
Size: 1<<32 - 1,
Modified: time.Date(2017, 10, 31, 21, 11, 58, 0, time.UTC),
Mode: 0666,
},
},
},
{
Name: "time-winrar.zip",
File: []ZipTestFile{
{
Name: "test.txt",
Content: []byte{},
Size: 1<<32 - 1,
Modified: time.Date(2017, 10, 31, 21, 11, 57, 244817900, timeZone(-7*time.Hour)),
Mode: 0666,
},
},
},
{
Name: "time-winzip.zip",
File: []ZipTestFile{
{
Name: "test.txt",
Content: []byte{},
Size: 1<<32 - 1,
Modified: time.Date(2017, 10, 31, 21, 11, 57, 244000000, timeZone(-7*time.Hour)),
Mode: 0666,
},
},
},
{
Name: "time-go.zip",
File: []ZipTestFile{
{
Name: "test.txt",
Content: []byte{},
Size: 1<<32 - 1,
Modified: time.Date(2017, 10, 31, 21, 11, 57, 0, timeZone(-7*time.Hour)),
Mode: 0666,
},
},
},
{
Name: "time-22738.zip",
File: []ZipTestFile{
{
Name: "file",
Content: []byte{},
Mode: 0666,
Modified: time.Date(1999, 12, 31, 19, 0, 0, 0, timeZone(-5*time.Hour)),
ModTime: time.Date(1999, 12, 31, 19, 0, 0, 0, time.UTC),
},
},
},
}
var crossPlatform = []ZipTestFile{
{
Name: "hello",
Content: []byte("world \r\n"),
Mode: 0666,
},
{
Name: "dir/bar",
Content: []byte("foo \r\n"),
Mode: 0666,
},
{
Name: "dir/empty/",
Content: []byte{},
Mode: os.ModeDir | 0777,
},
{
Name: "readonly",
Content: []byte("important \r\n"),
Mode: 0444,
},
}
func TestReader(t *testing.T) {
for _, zt := range tests {
readTestZip(t, zt)
t.Run(zt.Name, func(t *testing.T) {
readTestZip(t, zt)
})
}
}
@ -319,7 +504,7 @@ func readTestZip(t *testing.T, zt ZipTest) {
}
}
if err != zt.Error {
t.Errorf("%s: error=%v, want %v", zt.Name, err, zt.Error)
t.Errorf("error=%v, want %v", err, zt.Error)
return
}
@ -335,16 +520,19 @@ func readTestZip(t *testing.T, zt ZipTest) {
}
if z.Comment != zt.Comment {
t.Errorf("%s: comment=%q, want %q", zt.Name, z.Comment, zt.Comment)
t.Errorf("comment=%q, want %q", z.Comment, zt.Comment)
}
if len(z.File) != len(zt.File) {
t.Fatalf("%s: file count=%d, want %d", zt.Name, len(z.File), len(zt.File))
t.Fatalf("file count=%d, want %d", len(z.File), len(zt.File))
}
// test read of each file
for i, ft := range zt.File {
readTestFile(t, zt, ft, z.File[i])
}
if t.Failed() {
return
}
// test simultaneous reads
n := 0
@ -363,23 +551,24 @@ func readTestZip(t *testing.T, zt ZipTest) {
}
}
func equalTimeAndZone(t1, t2 time.Time) bool {
name1, offset1 := t1.Zone()
name2, offset2 := t2.Zone()
return t1.Equal(t2) && name1 == name2 && offset1 == offset2
}
func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
if f.Name != ft.Name {
t.Errorf("%s: name=%q, want %q", zt.Name, f.Name, ft.Name)
t.Errorf("name=%q, want %q", f.Name, ft.Name)
}
if !ft.Modified.IsZero() && !equalTimeAndZone(f.Modified, ft.Modified) {
t.Errorf("%s: Modified=%s, want %s", f.Name, f.Modified, ft.Modified)
}
if !ft.ModTime.IsZero() && !equalTimeAndZone(f.ModTime(), ft.ModTime) {
t.Errorf("%s: ModTime=%s, want %s", f.Name, f.ModTime(), ft.ModTime)
}
if ft.Mtime != "" {
mtime, err := time.Parse("01-02-06 15:04:05", ft.Mtime)
if err != nil {
t.Error(err)
return
}
if ft := f.ModTime(); !ft.Equal(mtime) {
t.Errorf("%s: %s: mtime=%s, want %s", zt.Name, f.Name, ft, mtime)
}
}
testFileMode(t, zt.Name, f, ft.Mode)
testFileMode(t, f, ft.Mode)
size := uint64(f.UncompressedSize)
if size == uint32max {
@ -390,7 +579,7 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
r, err := f.Open()
if err != nil {
t.Errorf("%s: %v", zt.Name, err)
t.Errorf("%v", err)
return
}
@ -408,7 +597,7 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
var b bytes.Buffer
_, err = io.Copy(&b, r)
if err != ft.ContentErr {
t.Errorf("%s: copying contents: %v (want %v)", zt.Name, err, ft.ContentErr)
t.Errorf("copying contents: %v (want %v)", err, ft.ContentErr)
}
if err != nil {
return
@ -440,12 +629,12 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
}
}
func testFileMode(t *testing.T, zipName string, f *File, want os.FileMode) {
func testFileMode(t *testing.T, f *File, want os.FileMode) {
mode := f.Mode()
if want == 0 {
t.Errorf("%s: %s mode: got %v, want none", zipName, f.Name, mode)
t.Errorf("%s mode: got %v, want none", f.Name, mode)
} else if mode != want {
t.Errorf("%s: %s mode: want %v, got %v", zipName, f.Name, want, mode)
t.Errorf("%s mode: want %v, got %v", f.Name, want, mode)
}
}

104
libgo/go/archive/zip/struct.go
View file

@ -27,8 +27,8 @@ import (
// Compression methods.
const (
Store uint16 = 0
Deflate uint16 = 8
Store uint16 = 0 // no compression
Deflate uint16 = 8 // DEFLATE compressed
)
const (
@ -46,40 +46,79 @@ const (
directory64LocLen = 20 //
directory64EndLen = 56 // + extra
// Constants for the first byte in CreatorVersion
// Constants for the first byte in CreatorVersion.
creatorFAT = 0
creatorUnix = 3
creatorNTFS = 11
creatorVFAT = 14
creatorMacOSX = 19
// version numbers
// Version numbers.
zipVersion20 = 20 // 2.0
zipVersion45 = 45 // 4.5 (reads and writes zip64 archives)
// limits for non zip64 files
// Limits for non zip64 files.
uint16max = (1 << 16) - 1
uint32max = (1 << 32) - 1
// extra header id's
zip64ExtraId = 0x0001 // zip64 Extended Information Extra Field
// Extra header IDs.
//
// IDs 0..31 are reserved for official use by PKWARE.
// IDs above that range are defined by third-party vendors.
// Since ZIP lacked high precision timestamps (nor a official specification
// of the timezone used for the date fields), many competing extra fields
// have been invented. Pervasive use effectively makes them "official".
//
// See http://mdfs.net/Docs/Comp/Archiving/Zip/ExtraField
zip64ExtraID = 0x0001 // Zip64 extended information
ntfsExtraID = 0x000a // NTFS
unixExtraID = 0x000d // UNIX
extTimeExtraID = 0x5455 // Extended timestamp
infoZipUnixExtraID = 0x5855 // Info-ZIP Unix extension
)
// FileHeader describes a file within a zip file.
// See the zip spec for details.
type FileHeader struct {
// Name is the name of the file.
// It must be a relative path: it must not start with a drive
// letter (e.g. C:) or leading slash, and only forward slashes
// are allowed.
// It must be a relative path, not start with a drive letter (e.g. C:),
// and must use forward slashes instead of back slashes.
Name string
CreatorVersion uint16
ReaderVersion uint16
Flags uint16
Method uint16
ModifiedTime uint16 // MS-DOS time
ModifiedDate uint16 // MS-DOS date
// Comment is any arbitrary user-defined string shorter than 64KiB.
Comment string
// NonUTF8 indicates that Name and Comment are not encoded in UTF-8.
//
// By specification, the only other encoding permitted should be CP-437,
// but historically many ZIP readers interpret Name and Comment as whatever
// the system's local character encoding happens to be.
//
// This flag should only be set if the user intends to encode a non-portable
// ZIP file for a specific localized region. Otherwise, the Writer
// automatically sets the ZIP format's UTF-8 flag for valid UTF-8 strings.
NonUTF8 bool
CreatorVersion uint16
ReaderVersion uint16
Flags uint16
// Method is the compression method. If zero, Store is used.
Method uint16
// Modified is the modified time of the file.
//
// When reading, an extended timestamp is preferred over the legacy MS-DOS
// date field, and the offset between the times is used as the timezone.
// If only the MS-DOS date is present, the timezone is assumed to be UTC.
//
// When writing, an extended timestamp (which is timezone-agnostic) is
// always emitted. The legacy MS-DOS date field is encoded according to the
// location of the Modified time.
Modified time.Time
ModifiedTime uint16 // Deprecated: Legacy MS-DOS date; use Modified instead.
ModifiedDate uint16 // Deprecated: Legacy MS-DOS time; use Modified instead.
CRC32 uint32
CompressedSize uint32 // Deprecated: Use CompressedSize64 instead.
UncompressedSize uint32 // Deprecated: Use UncompressedSize64 instead.
@ -87,7 +126,6 @@ type FileHeader struct {
UncompressedSize64 uint64
Extra []byte
ExternalAttrs uint32 // Meaning depends on CreatorVersion
Comment string
}
// FileInfo returns an os.FileInfo for the FileHeader.
@ -117,6 +155,8 @@ func (fi headerFileInfo) Sys() interface{} { return fi.fh }
// Because os.FileInfo's Name method returns only the base name of
// the file it describes, it may be necessary to modify the Name field
// of the returned header to provide the full path name of the file.
// If compression is desired, callers should set the FileHeader.Method
// field; it is unset by default.
func FileInfoHeader(fi os.FileInfo) (*FileHeader, error) {
size := fi.Size()
fh := &FileHeader{
@ -144,6 +184,21 @@ type directoryEnd struct {
comment string
}
// timeZone returns a *time.Location based on the provided offset.
// If the offset is non-sensible, then this uses an offset of zero.
func timeZone(offset time.Duration) *time.Location {
const (
minOffset = -12 * time.Hour // E.g., Baker island at -12:00
maxOffset = +14 * time.Hour // E.g., Line island at +14:00
offsetAlias = 15 * time.Minute // E.g., Nepal at +5:45
)
offset = offset.Round(offsetAlias)
if offset < minOffset || maxOffset < offset {
offset = 0
}
return time.FixedZone("", int(offset/time.Second))
}
// msDosTimeToTime converts an MS-DOS date and time into a time.Time.
// The resolution is 2s.
// See: http://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx
@ -168,21 +223,26 @@ func msDosTimeToTime(dosDate, dosTime uint16) time.Time {
// The resolution is 2s.
// See: http://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx
func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) {
t = t.In(time.UTC)
fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)
fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)
return
}
// ModTime returns the modification time in UTC.
// The resolution is 2s.
// ModTime returns the modification time in UTC using the legacy
// ModifiedDate and ModifiedTime fields.
//
// Deprecated: Use Modified instead.
func (h *FileHeader) ModTime() time.Time {
return msDosTimeToTime(h.ModifiedDate, h.ModifiedTime)
}
// SetModTime sets the ModifiedTime and ModifiedDate fields to the given time in UTC.
// The resolution is 2s.
// SetModTime sets the Modified, ModifiedTime, and ModifiedDate fields
// to the given time in UTC.
//
// Deprecated: Use Modified instead.
func (h *FileHeader) SetModTime(t time.Time) {
t = t.UTC() // Convert to UTC for compatibility
h.Modified = t
h.ModifiedDate, h.ModifiedTime = timeToMsDosTime(t)
}

BIN
libgo/go/archive/zip/testdata/time-22738.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/time-7zip.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/time-go.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/time-infozip.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/time-osx.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/time-win7.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/time-winrar.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/time-winzip.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/utf8-7zip.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/utf8-infozip.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/utf8-osx.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/utf8-winrar.zip vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/zip/testdata/utf8-winzip.zip vendored Normal file
View file

Binary file not shown.

137
libgo/go/archive/zip/writer.go
View file

@ -14,6 +14,11 @@ import (
"unicode/utf8"
)
var (
errLongName = errors.New("zip: FileHeader.Name too long")
errLongExtra = errors.New("zip: FileHeader.Extra too long")
)
// Writer implements a zip file writer.
type Writer struct {
cw *countWriter
@ -21,6 +26,7 @@ type Writer struct {
last *fileWriter
closed bool
compressors map[uint16]Compressor
comment string
// testHookCloseSizeOffset if non-nil is called with the size
// of offset of the central directory at Close.
@ -54,6 +60,16 @@ func (w *Writer) Flush() error {
return w.cw.w.(*bufio.Writer).Flush()
}
// SetComment sets the end-of-central-directory comment field.
// It can only be called before Close.
func (w *Writer) SetComment(comment string) error {
if len(comment) > uint16max {
return errors.New("zip: Writer.Comment too long")
}
w.comment = comment
return nil
}
// Close finishes writing the zip file by writing the central directory.
// It does not (and cannot) close the underlying writer.
func (w *Writer) Close() error {
@ -91,7 +107,7 @@ func (w *Writer) Close() error {
// append a zip64 extra block to Extra
var buf [28]byte // 2x uint16 + 3x uint64
eb := writeBuf(buf[:])
eb.uint16(zip64ExtraId)
eb.uint16(zip64ExtraID)
eb.uint16(24) // size = 3x uint64
eb.uint64(h.UncompressedSize64)
eb.uint64(h.CompressedSize64)
@ -172,21 +188,25 @@ func (w *Writer) Close() error {
var buf [directoryEndLen]byte
b := writeBuf(buf[:])
b.uint32(uint32(directoryEndSignature))
b = b[4:] // skip over disk number and first disk number (2x uint16)
b.uint16(uint16(records)) // number of entries this disk
b.uint16(uint16(records)) // number of entries total
b.uint32(uint32(size)) // size of directory
b.uint32(uint32(offset)) // start of directory
// skipped size of comment (always zero)
b = b[4:] // skip over disk number and first disk number (2x uint16)
b.uint16(uint16(records)) // number of entries this disk
b.uint16(uint16(records)) // number of entries total
b.uint32(uint32(size)) // size of directory
b.uint32(uint32(offset)) // start of directory
b.uint16(uint16(len(w.comment))) // byte size of EOCD comment
if _, err := w.cw.Write(buf[:]); err != nil {
return err
}
if _, err := io.WriteString(w.cw, w.comment); err != nil {
return err
}
return w.cw.w.(*bufio.Writer).Flush()
}
// Create adds a file to the zip file using the provided name.
// It returns a Writer to which the file contents should be written.
// The file contents will be compressed using the Deflate method.
// The name must be a relative path: it must not start with a drive
// letter (e.g. C:) or leading slash, and only forward slashes are
// allowed.
@ -200,27 +220,36 @@ func (w *Writer) Create(name string) (io.Writer, error) {
return w.CreateHeader(header)
}
func hasValidUTF8(s string) bool {
n := 0
for _, r := range s {
// By default, ZIP uses CP437, which is only identical to ASCII for the printable characters.
if r < 0x20 || r >= 0x7f {
if !utf8.ValidRune(r) {
return false
// detectUTF8 reports whether s is a valid UTF-8 string, and whether the string
// must be considered UTF-8 encoding (i.e., not compatible with CP-437, ASCII,
// or any other common encoding).
func detectUTF8(s string) (valid, require bool) {
for i := 0; i < len(s); {
r, size := utf8.DecodeRuneInString(s[i:])
i += size
// Officially, ZIP uses CP-437, but many readers use the system's
// local character encoding. Most encoding are compatible with a large
// subset of CP-437, which itself is ASCII-like.
//
// Forbid 0x7e and 0x5c since EUC-KR and Shift-JIS replace those
// characters with localized currency and overline characters.
if r < 0x20 || r > 0x7d || r == 0x5c {
if !utf8.ValidRune(r) || (r == utf8.RuneError && size == 1) {
return false, false
}
n++
require = true
}
}
return n > 0
return true, require
}
// CreateHeader adds a file to the zip file using the provided FileHeader
// for the file metadata.
// It returns a Writer to which the file contents should be written.
// CreateHeader adds a file to the zip archive using the provided FileHeader
// for the file metadata. Writer takes ownership of fh and may mutate
// its fields. The caller must not modify fh after calling CreateHeader.
//
// This returns a Writer to which the file contents should be written.
// The file's contents must be written to the io.Writer before the next
// call to Create, CreateHeader, or Close. The provided FileHeader fh
// must not be modified after a call to CreateHeader.
// call to Create, CreateHeader, or Close.
func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
if w.last != nil && !w.last.closed {
if err := w.last.close(); err != nil {
@ -234,13 +263,62 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
fh.Flags |= 0x8 // we will write a data descriptor
if hasValidUTF8(fh.Name) || hasValidUTF8(fh.Comment) {
fh.Flags |= 0x800 // filename or comment have valid utf-8 string
// The ZIP format has a sad state of affairs regarding character encoding.
// Officially, the name and comment fields are supposed to be encoded
// in CP-437 (which is mostly compatible with ASCII), unless the UTF-8
// flag bit is set. However, there are several problems:
//
// * Many ZIP readers still do not support UTF-8.
// * If the UTF-8 flag is cleared, several readers simply interpret the
// name and comment fields as whatever the local system encoding is.
//
// In order to avoid breaking readers without UTF-8 support,
// we avoid setting the UTF-8 flag if the strings are CP-437 compatible.
// However, if the strings require multibyte UTF-8 encoding and is a
// valid UTF-8 string, then we set the UTF-8 bit.
//
// For the case, where the user explicitly wants to specify the encoding
// as UTF-8, they will need to set the flag bit themselves.
utf8Valid1, utf8Require1 := detectUTF8(fh.Name)
utf8Valid2, utf8Require2 := detectUTF8(fh.Comment)
switch {
case fh.NonUTF8:
fh.Flags &^= 0x800
case (utf8Require1 || utf8Require2) && (utf8Valid1 && utf8Valid2):
fh.Flags |= 0x800
}
fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte
fh.ReaderVersion = zipVersion20
// If Modified is set, this takes precedence over MS-DOS timestamp fields.
if !fh.Modified.IsZero() {
// Contrary to the FileHeader.SetModTime method, we intentionally
// do not convert to UTC, because we assume the user intends to encode
// the date using the specified timezone. A user may want this control
// because many legacy ZIP readers interpret the timestamp according
// to the local timezone.
//
// The timezone is only non-UTC if a user directly sets the Modified
// field directly themselves. All other approaches sets UTC.
fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(fh.Modified)
// Use "extended timestamp" format since this is what Info-ZIP uses.
// Nearly every major ZIP implementation uses a different format,
// but at least most seem to be able to understand the other formats.
//
// This format happens to be identical for both local and central header
// if modification time is the only timestamp being encoded.
var mbuf [9]byte // 2*SizeOf(uint16) + SizeOf(uint8) + SizeOf(uint32)
mt := uint32(fh.Modified.Unix())
eb := writeBuf(mbuf[:])
eb.uint16(extTimeExtraID)
eb.uint16(5) // Size: SizeOf(uint8) + SizeOf(uint32)
eb.uint8(1) // Flags: ModTime
eb.uint32(mt) // ModTime
fh.Extra = append(fh.Extra, mbuf[:]...)
}
fw := &fileWriter{
zipw: w.cw,
compCount: &countWriter{w: w.cw},
@ -273,6 +351,14 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
}
func writeHeader(w io.Writer, h *FileHeader) error {
const maxUint16 = 1<<16 - 1
if len(h.Name) > maxUint16 {
return errLongName
}
if len(h.Extra) > maxUint16 {
return errLongExtra
}
var buf [fileHeaderLen]byte
b := writeBuf(buf[:])
b.uint32(uint32(fileHeaderSignature))
@ -402,6 +488,11 @@ func (w nopCloser) Close() error {
type writeBuf []byte
func (b *writeBuf) uint8(v uint8) {
(*b)[0] = v
*b = (*b)[1:]
}
func (b *writeBuf) uint16(v uint16) {
binary.LittleEndian.PutUint16(*b, v)
*b = (*b)[2:]

124
libgo/go/archive/zip/writer_test.go
View file

@ -6,11 +6,14 @@ package zip
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"strings"
"testing"
"time"
)
// TODO(adg): a more sophisticated test suite
@ -57,8 +60,8 @@ var writeTests = []WriteTest{
func TestWriter(t *testing.T) {
largeData := make([]byte, 1<<17)
for i := range largeData {
largeData[i] = byte(rand.Int())
if _, err := rand.Read(largeData); err != nil {
t.Fatal("rand.Read failed:", err)
}
writeTests[1].Data = largeData
defer func() {
@ -87,31 +90,100 @@ func TestWriter(t *testing.T) {
}
}
// TestWriterComment is test for EOCD comment read/write.
func TestWriterComment(t *testing.T) {
var tests = []struct {
comment string
ok bool
}{
{"hi, hello", true},
{"hi, こんにちわ", true},
{strings.Repeat("a", uint16max), true},
{strings.Repeat("a", uint16max+1), false},
}
for _, test := range tests {
// write a zip file
buf := new(bytes.Buffer)
w := NewWriter(buf)
if err := w.SetComment(test.comment); err != nil {
if test.ok {
t.Fatalf("SetComment: unexpected error %v", err)
}
continue
} else {
if !test.ok {
t.Fatalf("SetComment: unexpected success, want error")
}
}
if err := w.Close(); test.ok == (err != nil) {
t.Fatal(err)
}
if w.closed != test.ok {
t.Fatalf("Writer.closed: got %v, want %v", w.closed, test.ok)
}
// skip read test in failure cases
if !test.ok {
continue
}
// read it back
r, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
if err != nil {
t.Fatal(err)
}
if r.Comment != test.comment {
t.Fatalf("Reader.Comment: got %v, want %v", r.Comment, test.comment)
}
}
}
func TestWriterUTF8(t *testing.T) {
var utf8Tests = []struct {
name string
comment string
expect uint16
nonUTF8 bool
flags uint16
}{
{
name: "hi, hello",
comment: "in the world",
expect: 0x8,
flags: 0x8,
},
{
name: "hi, こんにちわ",
comment: "in the world",
expect: 0x808,
flags: 0x808,
},
{
name: "hi, こんにちわ",
comment: "in the world",
nonUTF8: true,
flags: 0x8,
},
{
name: "hi, hello",
comment: "in the 世界",
expect: 0x808,
flags: 0x808,
},
{
name: "hi, こんにちわ",
comment: "in the 世界",
expect: 0x808,
flags: 0x808,
},
{
name: "the replacement rune is <20>",
comment: "the replacement rune is <20>",
flags: 0x808,
},
{
// Name is Japanese encoded in Shift JIS.
name: "\x93\xfa\x96{\x8c\xea.txt",
comment: "in the 世界",
flags: 0x008, // UTF-8 must not be set
},
}
@ -123,6 +195,7 @@ func TestWriterUTF8(t *testing.T) {
h := &FileHeader{
Name: test.name,
Comment: test.comment,
NonUTF8: test.nonUTF8,
Method: Deflate,
}
w, err := w.CreateHeader(h)
@ -142,18 +215,41 @@ func TestWriterUTF8(t *testing.T) {
t.Fatal(err)
}
for i, test := range utf8Tests {
got := r.File[i].Flags
t.Logf("name %v, comment %v", test.name, test.comment)
if got != test.expect {
t.Fatalf("Flags: got %v, want %v", got, test.expect)
flags := r.File[i].Flags
if flags != test.flags {
t.Errorf("CreateHeader(name=%q comment=%q nonUTF8=%v): flags=%#x, want %#x", test.name, test.comment, test.nonUTF8, flags, test.flags)
}
}
}
func TestWriterTime(t *testing.T) {
var buf bytes.Buffer
h := &FileHeader{
Name: "test.txt",
Modified: time.Date(2017, 10, 31, 21, 11, 57, 0, timeZone(-7*time.Hour)),
}
w := NewWriter(&buf)
if _, err := w.CreateHeader(h); err != nil {
t.Fatalf("unexpected CreateHeader error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected Close error: %v", err)
}
want, err := ioutil.ReadFile("testdata/time-go.zip")
if err != nil {
t.Fatalf("unexpected ReadFile error: %v", err)
}
if got := buf.Bytes(); !bytes.Equal(got, want) {
fmt.Printf("%x\n%x\n", got, want)
t.Error("contents of time-go.zip differ")
}
}
func TestWriterOffset(t *testing.T) {
largeData := make([]byte, 1<<17)
for i := range largeData {
largeData[i] = byte(rand.Int())
if _, err := rand.Read(largeData); err != nil {
t.Fatal("rand.Read failed:", err)
}
writeTests[1].Data = largeData
defer func() {
@ -225,7 +321,7 @@ func testReadFile(t *testing.T, f *File, wt *WriteTest) {
if f.Name != wt.Name {
t.Fatalf("File name: got %q, want %q", f.Name, wt.Name)
}
testFileMode(t, wt.Name, f, wt.Mode)
testFileMode(t, f, wt.Mode)
rc, err := f.Open()
if err != nil {
t.Fatal("opening:", err)

42
libgo/go/archive/zip/zip_test.go
View file

@ -645,16 +645,54 @@ func TestHeaderTooShort(t *testing.T) {
h := FileHeader{
Name: "foo.txt",
Method: Deflate,
Extra: []byte{zip64ExtraId}, // missing size and second half of tag, but Extra is best-effort parsing
Extra: []byte{zip64ExtraID}, // missing size and second half of tag, but Extra is best-effort parsing
}
testValidHeader(&h, t)
}
func TestHeaderTooLongErr(t *testing.T) {
var headerTests = []struct {
name string
extra []byte
wanterr error
}{
{
name: strings.Repeat("x", 1<<16),
extra: []byte{},
wanterr: errLongName,
},
{
name: "long_extra",
extra: bytes.Repeat([]byte{0xff}, 1<<16),
wanterr: errLongExtra,
},
}
// write a zip file
buf := new(bytes.Buffer)
w := NewWriter(buf)
for _, test := range headerTests {
h := &FileHeader{
Name: test.name,
Extra: test.extra,
}
_, err := w.CreateHeader(h)
if err != test.wanterr {
t.Errorf("error=%v, want %v", err, test.wanterr)
}
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
}
func TestHeaderIgnoredSize(t *testing.T) {
h := FileHeader{
Name: "foo.txt",
Method: Deflate,
Extra: []byte{zip64ExtraId & 0xFF, zip64ExtraId >> 8, 24, 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8}, // bad size but shouldn't be consulted
Extra: []byte{zip64ExtraID & 0xFF, zip64ExtraID >> 8, 24, 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8}, // bad size but shouldn't be consulted
}
testValidHeader(&h, t)
}

6
libgo/go/bufio/bufio.go
View file

@ -62,6 +62,9 @@ func NewReader(rd io.Reader) *Reader {
return NewReaderSize(rd, defaultBufSize)
}
// Size returns the size of the underlying buffer in bytes.
func (r *Reader) Size() int { return len(r.buf) }
// Reset discards any buffered data, resets all state, and switches
// the buffered reader to read from r.
func (b *Reader) Reset(r io.Reader) {
@ -548,6 +551,9 @@ func NewWriter(w io.Writer) *Writer {
return NewWriterSize(w, defaultBufSize)
}
// Size returns the size of the underlying buffer in bytes.
func (b *Writer) Size() int { return len(b.buf) }
// Reset discards any unflushed buffered data, clears any error, and
// resets b to write its output to w.
func (b *Writer) Reset(w io.Writer) {

18
libgo/go/bufio/bufio_test.go
View file

@ -1418,6 +1418,24 @@ func TestReaderDiscard(t *testing.T) {
}
func TestReaderSize(t *testing.T) {
if got, want := NewReader(nil).Size(), DefaultBufSize; got != want {
t.Errorf("NewReader's Reader.Size = %d; want %d", got, want)
}
if got, want := NewReaderSize(nil, 1234).Size(), 1234; got != want {
t.Errorf("NewReaderSize's Reader.Size = %d; want %d", got, want)
}
}
func TestWriterSize(t *testing.T) {
if got, want := NewWriter(nil).Size(), DefaultBufSize; got != want {
t.Errorf("NewWriter's Writer.Size = %d; want %d", got, want)
}
if got, want := NewWriterSize(nil, 1234).Size(), 1234; got != want {
t.Errorf("NewWriterSize's Writer.Size = %d; want %d", got, want)
}
}
// An onlyReader only implements io.Reader, no matter what other methods the underlying implementation may have.
type onlyReader struct {
io.Reader

2
libgo/go/bufio/export_test.go
View file

@ -11,6 +11,8 @@ import (
var IsSpace = isSpace
const DefaultBufSize = defaultBufSize
func (s *Scanner) MaxTokenSize(n int) {
if n < utf8.UTFMax || n > 1e9 {
panic("bad max token size")

9
libgo/go/bufio/scan.go
View file

@ -123,8 +123,9 @@ var ErrFinalToken = errors.New("final token")
// After Scan returns false, the Err method will return any error that
// occurred during scanning, except that if it was io.EOF, Err
// will return nil.
// Scan panics if the split function returns 100 empty tokens without
// advancing the input. This is a common error mode for scanners.
// Scan panics if the split function returns too many empty
// tokens without advancing the input. This is a common error mode for
// scanners.
func (s *Scanner) Scan() bool {
if s.done {
return false
@ -156,8 +157,8 @@ func (s *Scanner) Scan() bool {
} else {
// Returning tokens not advancing input at EOF.
s.empties++
if s.empties > 100 {
panic("bufio.Scan: 100 empty tokens without progressing")
if s.empties > maxConsecutiveEmptyReads {
panic("bufio.Scan: too many empty tokens without progressing")
}
}
return true

5
libgo/go/builtin/builtin.go
View file

@ -171,8 +171,9 @@ func cap(v Type) int
// Slice: The size specifies the length. The capacity of the slice is
// equal to its length. A second integer argument may be provided to
// specify a different capacity; it must be no smaller than the
// length, so make([]int, 0, 10) allocates a slice of length 0 and
// capacity 10.
// length. For example, make([]int, 0, 10) allocates an underlying array
// of size 10 and returns a slice of length 0 and capacity 10 that is
// backed by this underlying array.
// Map: An empty map is allocated with enough space to hold the
// specified number of elements. The size may be omitted, in which case
// a small starting size is allocated.

84
libgo/go/bytes/boundary_test.go Normal file
View file

@ -0,0 +1,84 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// +build linux
package bytes_test
import (
. "bytes"
"syscall"
"testing"
)
// This file tests the situation where byte operations are checking
// data very near to a page boundary. We want to make sure those
// operations do not read across the boundary and cause a page
// fault where they shouldn't.
// These tests run only on linux. The code being tested is
// not OS-specific, so it does not need to be tested on all
// operating systems.
// dangerousSlice returns a slice which is immediately
// preceded and followed by a faulting page.
func dangerousSlice(t *testing.T) []byte {
pagesize := syscall.Getpagesize()
b, err := syscall.Mmap(0, 0, 3*pagesize, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANONYMOUS|syscall.MAP_PRIVATE)
if err != nil {
t.Fatalf("mmap failed %s", err)
}
err = syscall.Mprotect(b[:pagesize], syscall.PROT_NONE)
if err != nil {
t.Fatalf("mprotect low failed %s\n", err)
}
err = syscall.Mprotect(b[2*pagesize:], syscall.PROT_NONE)
if err != nil {
t.Fatalf("mprotect high failed %s\n", err)
}
return b[pagesize : 2*pagesize]
}
func TestEqualNearPageBoundary(t *testing.T) {
t.Parallel()
b := dangerousSlice(t)
for i := range b {
b[i] = 'A'
}
for i := 0; i <= len(b); i++ {
Equal(b[:i], b[len(b)-i:])
Equal(b[len(b)-i:], b[:i])
}
}
func TestIndexByteNearPageBoundary(t *testing.T) {
t.Parallel()
b := dangerousSlice(t)
for i := range b {
idx := IndexByte(b[i:], 1)
if idx != -1 {
t.Fatalf("IndexByte(b[%d:])=%d, want -1\n", i, idx)
}
}
}
func TestIndexNearPageBoundary(t *testing.T) {
t.Parallel()
var q [64]byte
b := dangerousSlice(t)
if len(b) > 256 {
// Only worry about when we're near the end of a page.
b = b[len(b)-256:]
}
for j := 1; j < len(q); j++ {
q[j-1] = 1 // difference is only found on the last byte
for i := range b {
idx := Index(b[i:], q[:j])
if idx != -1 {
t.Fatalf("Index(b[%d:], q[:%d])=%d, want -1\n", i, j, idx)
}
}
q[j-1] = 0
}
}

94
libgo/go/bytes/buffer.go
View file

@ -15,34 +15,37 @@ import (
// A Buffer is a variable-sized buffer of bytes with Read and Write methods.
// The zero value for Buffer is an empty buffer ready to use.
type Buffer struct {
buf []byte // contents are the bytes buf[off : len(buf)]
off int // read at &buf[off], write at &buf[len(buf)]
lastRead readOp // last read operation, so that Unread* can work correctly.
// FIXME: lastRead can fit in a single byte
buf []byte // contents are the bytes buf[off : len(buf)]
off int // read at &buf[off], write at &buf[len(buf)]
bootstrap [64]byte // memory to hold first slice; helps small buffers avoid allocation.
lastRead readOp // last read operation, so that Unread* can work correctly.
// memory to hold first slice; helps small buffers avoid allocation.
// FIXME: it would be advisable to align Buffer to cachelines to avoid false
// sharing.
bootstrap [64]byte
}
// The readOp constants describe the last action performed on
// the buffer, so that UnreadRune and UnreadByte can check for
// invalid usage. opReadRuneX constants are chosen such that
// converted to int they correspond to the rune size that was read.
type readOp int
type readOp int8
// Don't use iota for these, as the values need to correspond with the
// names and comments, which is easier to see when being explicit.
const (
opRead readOp = -1 // Any other read operation.
opInvalid = 0 // Non-read operation.
opReadRune1 = 1 // Read rune of size 1.
opReadRune2 = 2 // Read rune of size 2.
opReadRune3 = 3 // Read rune of size 3.
opReadRune4 = 4 // Read rune of size 4.
opInvalid readOp = 0 // Non-read operation.
opReadRune1 readOp = 1 // Read rune of size 1.
opReadRune2 readOp = 2 // Read rune of size 2.
opReadRune3 readOp = 3 // Read rune of size 3.
opReadRune4 readOp = 4 // Read rune of size 4.
)
// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
var ErrTooLarge = errors.New("bytes.Buffer: too large")
var errNegativeRead = errors.New("bytes.Buffer: reader returned negative count from Read")
const maxInt = int(^uint(0) >> 1)
// Bytes returns a slice of length b.Len() holding the unread portion of the buffer.
// The slice is valid for use only until the next buffer modification (that is,
@ -53,6 +56,8 @@ func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }
// String returns the contents of the unread portion of the buffer
// as a string. If the Buffer is a nil pointer, it returns "<nil>".
//
// To build strings more efficiently, see the strings.Builder type.
func (b *Buffer) String() string {
if b == nil {
// Special case, useful in debugging.
@ -61,6 +66,9 @@ func (b *Buffer) String() string {
return string(b.buf[b.off:])
}
// empty returns whether the unread portion of the buffer is empty.
func (b *Buffer) empty() bool { return len(b.buf) <= b.off }
// Len returns the number of bytes of the unread portion of the buffer;
// b.Len() == len(b.Bytes()).
func (b *Buffer) Len() int { return len(b.buf) - b.off }
@ -81,7 +89,7 @@ func (b *Buffer) Truncate(n int) {
if n < 0 || n > b.Len() {
panic("bytes.Buffer: truncation out of range")
}
b.buf = b.buf[0 : b.off+n]
b.buf = b.buf[:b.off+n]
}
// Reset resets the buffer to be empty,
@ -97,7 +105,7 @@ func (b *Buffer) Reset() {
// internal buffer only needs to be resliced.
// It returns the index where bytes should be written and whether it succeeded.
func (b *Buffer) tryGrowByReslice(n int) (int, bool) {
if l := len(b.buf); l+n <= cap(b.buf) {
if l := len(b.buf); n <= cap(b.buf)-l {
b.buf = b.buf[:l+n]
return l, true
}
@ -122,15 +130,18 @@ func (b *Buffer) grow(n int) int {
b.buf = b.bootstrap[:n]
return 0
}
if m+n <= cap(b.buf)/2 {
c := cap(b.buf)
if n <= c/2-m {
// We can slide things down instead of allocating a new
// slice. We only need m+n <= cap(b.buf) to slide, but
// slice. We only need m+n <= c to slide, but
// we instead let capacity get twice as large so we
// don't spend all our time copying.
copy(b.buf[:], b.buf[b.off:])
copy(b.buf, b.buf[b.off:])
} else if c > maxInt-c-n {
panic(ErrTooLarge)
} else {
// Not enough space anywhere, we need to allocate.
buf := makeSlice(2*cap(b.buf) + n)
buf := makeSlice(2*c + n)
copy(buf, b.buf[b.off:])
b.buf = buf
}
@ -150,7 +161,7 @@ func (b *Buffer) Grow(n int) {
panic("bytes.Buffer.Grow: negative count")
}
m := b.grow(n)
b.buf = b.buf[0:m]
b.buf = b.buf[:m]
}
// Write appends the contents of p to the buffer, growing the buffer as
@ -189,34 +200,22 @@ const MinRead = 512
// buffer becomes too large, ReadFrom will panic with ErrTooLarge.
func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) {
b.lastRead = opInvalid
// If buffer is empty, reset to recover space.
if b.off >= len(b.buf) {
b.Reset()
}
for {
if free := cap(b.buf) - len(b.buf); free < MinRead {
// not enough space at end
newBuf := b.buf
if b.off+free < MinRead {
// not enough space using beginning of buffer;
// double buffer capacity
newBuf = makeSlice(2*cap(b.buf) + MinRead)
}
copy(newBuf, b.buf[b.off:])
b.buf = newBuf[:len(b.buf)-b.off]
b.off = 0
i := b.grow(MinRead)
m, e := r.Read(b.buf[i:cap(b.buf)])
if m < 0 {
panic(errNegativeRead)
}
m, e := r.Read(b.buf[len(b.buf):cap(b.buf)])
b.buf = b.buf[0 : len(b.buf)+m]
b.buf = b.buf[:i+m]
n += int64(m)
if e == io.EOF {
break
return n, nil // e is EOF, so return nil explicitly
}
if e != nil {
return n, e
}
}
return n, nil // err is EOF, so return nil explicitly
}
// makeSlice allocates a slice of size n. If the allocation fails, it panics
@ -237,8 +236,7 @@ func makeSlice(n int) []byte {
// encountered during the write is also returned.
func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
b.lastRead = opInvalid
if b.off < len(b.buf) {
nBytes := b.Len()
if nBytes := b.Len(); nBytes > 0 {
m, e := w.Write(b.buf[b.off:])
if m > nBytes {
panic("bytes.Buffer.WriteTo: invalid Write count")
@ -256,7 +254,7 @@ func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
}
// Buffer is now empty; reset.
b.Reset()
return
return n, nil
}
// WriteByte appends the byte c to the buffer, growing the buffer as needed.
@ -298,11 +296,11 @@ func (b *Buffer) WriteRune(r rune) (n int, err error) {
// otherwise it is nil.
func (b *Buffer) Read(p []byte) (n int, err error) {
b.lastRead = opInvalid
if b.off >= len(b.buf) {
if b.empty() {
// Buffer is empty, reset to recover space.
b.Reset()
if len(p) == 0 {
return
return 0, nil
}
return 0, io.EOF
}
@ -311,7 +309,7 @@ func (b *Buffer) Read(p []byte) (n int, err error) {
if n > 0 {
b.lastRead = opRead
}
return
return n, nil
}
// Next returns a slice containing the next n bytes from the buffer,
@ -335,8 +333,7 @@ func (b *Buffer) Next(n int) []byte {
// ReadByte reads and returns the next byte from the buffer.
// If no byte is available, it returns error io.EOF.
func (b *Buffer) ReadByte() (byte, error) {
b.lastRead = opInvalid
if b.off >= len(b.buf) {
if b.empty() {
// Buffer is empty, reset to recover space.
b.Reset()
return 0, io.EOF
@ -353,8 +350,7 @@ func (b *Buffer) ReadByte() (byte, error) {
// If the bytes are an erroneous UTF-8 encoding, it
// consumes one byte and returns U+FFFD, 1.
func (b *Buffer) ReadRune() (r rune, size int, err error) {
b.lastRead = opInvalid
if b.off >= len(b.buf) {
if b.empty() {
// Buffer is empty, reset to recover space.
b.Reset()
return 0, 0, io.EOF
@ -413,7 +409,7 @@ func (b *Buffer) ReadBytes(delim byte) (line []byte, err error) {
// return a copy of slice. The buffer's backing array may
// be overwritten by later calls.
line = append(line, slice...)
return
return line, err
}
// readSlice is like ReadBytes but returns a reference to internal buffer data.

114
libgo/go/bytes/buffer_test.go
View file

@ -6,25 +6,27 @@ package bytes_test
import (
. "bytes"
"internal/testenv"
"io"
"math/rand"
"os/exec"
"runtime"
"testing"
"unicode/utf8"
)
const N = 10000 // make this bigger for a larger (and slower) test
var data string // test data for write tests
var testBytes []byte // test data; same as data but as a slice.
const N = 10000 // make this bigger for a larger (and slower) test
var testString string // test data for write tests
var testBytes []byte // test data; same as testString but as a slice.
type negativeReader struct{}
func (r *negativeReader) Read([]byte) (int, error) { return -1, nil }
func init() {
testBytes = make([]byte, N)
for i := 0; i < N; i++ {
testBytes[i] = 'a' + byte(i%26)
}
data = string(testBytes)
testString = string(testBytes)
}
// Verify that contents of buf match the string s.
@ -88,12 +90,12 @@ func fillBytes(t *testing.T, testname string, buf *Buffer, s string, n int, fub
func TestNewBuffer(t *testing.T) {
buf := NewBuffer(testBytes)
check(t, "NewBuffer", buf, data)
check(t, "NewBuffer", buf, testString)
}
func TestNewBufferString(t *testing.T) {
buf := NewBufferString(data)
check(t, "NewBufferString", buf, data)
buf := NewBufferString(testString)
check(t, "NewBufferString", buf, testString)
}
// Empty buf through repeated reads into fub.
@ -128,7 +130,7 @@ func TestBasicOperations(t *testing.T) {
buf.Truncate(0)
check(t, "TestBasicOperations (3)", &buf, "")
n, err := buf.Write([]byte(data[0:1]))
n, err := buf.Write(testBytes[0:1])
if n != 1 {
t.Errorf("wrote 1 byte, but n == %d", n)
}
@ -137,30 +139,30 @@ func TestBasicOperations(t *testing.T) {
}
check(t, "TestBasicOperations (4)", &buf, "a")
buf.WriteByte(data[1])
buf.WriteByte(testString[1])
check(t, "TestBasicOperations (5)", &buf, "ab")
n, err = buf.Write([]byte(data[2:26]))
n, err = buf.Write(testBytes[2:26])
if n != 24 {
t.Errorf("wrote 25 bytes, but n == %d", n)
t.Errorf("wrote 24 bytes, but n == %d", n)
}
check(t, "TestBasicOperations (6)", &buf, string(data[0:26]))
check(t, "TestBasicOperations (6)", &buf, testString[0:26])
buf.Truncate(26)
check(t, "TestBasicOperations (7)", &buf, string(data[0:26]))
check(t, "TestBasicOperations (7)", &buf, testString[0:26])
buf.Truncate(20)
check(t, "TestBasicOperations (8)", &buf, string(data[0:20]))
check(t, "TestBasicOperations (8)", &buf, testString[0:20])
empty(t, "TestBasicOperations (9)", &buf, string(data[0:20]), make([]byte, 5))
empty(t, "TestBasicOperations (9)", &buf, testString[0:20], make([]byte, 5))
empty(t, "TestBasicOperations (10)", &buf, "", make([]byte, 100))
buf.WriteByte(data[1])
buf.WriteByte(testString[1])
c, err := buf.ReadByte()
if err != nil {
t.Error("ReadByte unexpected eof")
}
if c != data[1] {
if c != testString[1] {
t.Errorf("ReadByte wrong value c=%v", c)
}
c, err = buf.ReadByte()
@ -177,8 +179,8 @@ func TestLargeStringWrites(t *testing.T) {
limit = 9
}
for i := 3; i < limit; i += 3 {
s := fillString(t, "TestLargeWrites (1)", &buf, "", 5, data)
empty(t, "TestLargeStringWrites (2)", &buf, s, make([]byte, len(data)/i))
s := fillString(t, "TestLargeWrites (1)", &buf, "", 5, testString)
empty(t, "TestLargeStringWrites (2)", &buf, s, make([]byte, len(testString)/i))
}
check(t, "TestLargeStringWrites (3)", &buf, "")
}
@ -191,7 +193,7 @@ func TestLargeByteWrites(t *testing.T) {
}
for i := 3; i < limit; i += 3 {
s := fillBytes(t, "TestLargeWrites (1)", &buf, "", 5, testBytes)
empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(data)/i))
empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(testString)/i))
}
check(t, "TestLargeByteWrites (3)", &buf, "")
}
@ -199,8 +201,8 @@ func TestLargeByteWrites(t *testing.T) {
func TestLargeStringReads(t *testing.T) {
var buf Buffer
for i := 3; i < 30; i += 3 {
s := fillString(t, "TestLargeReads (1)", &buf, "", 5, data[0:len(data)/i])
empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data)))
s := fillString(t, "TestLargeReads (1)", &buf, "", 5, testString[0:len(testString)/i])
empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(testString)))
}
check(t, "TestLargeStringReads (3)", &buf, "")
}
@ -209,7 +211,7 @@ func TestLargeByteReads(t *testing.T) {
var buf Buffer
for i := 3; i < 30; i += 3 {
s := fillBytes(t, "TestLargeReads (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data)))
empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(testString)))
}
check(t, "TestLargeByteReads (3)", &buf, "")
}
@ -218,14 +220,14 @@ func TestMixedReadsAndWrites(t *testing.T) {
var buf Buffer
s := ""
for i := 0; i < 50; i++ {
wlen := rand.Intn(len(data))
wlen := rand.Intn(len(testString))
if i%2 == 0 {
s = fillString(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, data[0:wlen])
s = fillString(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, testString[0:wlen])
} else {
s = fillBytes(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, testBytes[0:wlen])
}
rlen := rand.Intn(len(data))
rlen := rand.Intn(len(testString))
fub := make([]byte, rlen)
n, _ := buf.Read(fub)
s = s[n:]
@ -263,17 +265,37 @@ func TestReadFrom(t *testing.T) {
s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
var b Buffer
b.ReadFrom(&buf)
empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data)))
empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(testString)))
}
}
func TestReadFromNegativeReader(t *testing.T) {
var b Buffer
defer func() {
switch err := recover().(type) {
case nil:
t.Fatal("bytes.Buffer.ReadFrom didn't panic")
case error:
// this is the error string of errNegativeRead
wantError := "bytes.Buffer: reader returned negative count from Read"
if err.Error() != wantError {
t.Fatalf("recovered panic: got %v, want %v", err.Error(), wantError)
}
default:
t.Fatalf("unexpected panic value: %#v", err)
}
}()
b.ReadFrom(new(negativeReader))
}
func TestWriteTo(t *testing.T) {
var buf Buffer
for i := 3; i < 30; i += 3 {
s := fillBytes(t, "TestWriteTo (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
var b Buffer
buf.WriteTo(&b)
empty(t, "TestWriteTo (2)", &b, s, make([]byte, len(data)))
empty(t, "TestWriteTo (2)", &b, s, make([]byte, len(testString)))
}
}
@ -473,6 +495,18 @@ func TestGrow(t *testing.T) {
}
}
func TestGrowOverflow(t *testing.T) {
defer func() {
if err := recover(); err != ErrTooLarge {
t.Errorf("after too-large Grow, recover() = %v; want %v", err, ErrTooLarge)
}
}()
buf := NewBuffer(make([]byte, 1))
const maxInt = int(^uint(0) >> 1)
buf.Grow(maxInt)
}
// Was a bug: used to give EOF reading empty slice at EOF.
func TestReadEmptyAtEOF(t *testing.T) {
b := new(Buffer)
@ -548,26 +582,6 @@ func TestBufferGrowth(t *testing.T) {
}
}
// Test that tryGrowByReslice is inlined.
// Only execute on "linux-amd64" builder in order to avoid breakage.
func TestTryGrowByResliceInlined(t *testing.T) {
targetBuilder := "linux-amd64"
if testenv.Builder() != targetBuilder {
t.Skipf("%q gets executed on %q builder only", t.Name(), targetBuilder)
}
t.Parallel()
goBin := testenv.GoToolPath(t)
out, err := exec.Command(goBin, "tool", "nm", goBin).CombinedOutput()
if err != nil {
t.Fatalf("go tool nm: %v: %s", err, out)
}
// Verify this doesn't exist:
sym := "bytes.(*Buffer).tryGrowByReslice"
if Contains(out, []byte(sym)) {
t.Errorf("found symbol %q in cmd/go, but should be inlined", sym)
}
}
func BenchmarkWriteByte(b *testing.B) {
const n = 4 << 10
b.SetBytes(n)

296
libgo/go/bytes/bytes.go
View file

@ -39,7 +39,7 @@ func explode(s []byte, n int) [][]byte {
break
}
_, size = utf8.DecodeRune(s)
a[na] = s[0:size]
a[na] = s[0:size:size]
s = s[size:]
na++
}
@ -68,12 +68,12 @@ func Contains(b, subslice []byte) bool {
return Index(b, subslice) != -1
}
// ContainsAny reports whether any of the UTF-8-encoded Unicode code points in chars are within b.
// ContainsAny reports whether any of the UTF-8-encoded code points in chars are within b.
func ContainsAny(b []byte, chars string) bool {
return IndexAny(b, chars) >= 0
}
// ContainsRune reports whether the Unicode code point r is within b.
// ContainsRune reports whether the rune is contained in the UTF-8-encoded byte slice b.
func ContainsRune(b []byte, r rune) bool {
return IndexRune(b, r) >= 0
}
@ -112,7 +112,7 @@ func LastIndexByte(s []byte, c byte) int {
return -1
}
// IndexRune interprets s as a sequence of UTF-8-encoded Unicode code points.
// IndexRune interprets s as a sequence of UTF-8-encoded code points.
// It returns the byte index of the first occurrence in s of the given rune.
// It returns -1 if rune is not present in s.
// If r is utf8.RuneError, it returns the first instance of any
@ -144,30 +144,32 @@ func IndexRune(s []byte, r rune) int {
// code points in chars. It returns -1 if chars is empty or if there is no code
// point in common.
func IndexAny(s []byte, chars string) int {
if len(chars) > 0 {
if len(s) > 8 {
if as, isASCII := makeASCIISet(chars); isASCII {
for i, c := range s {
if as.contains(c) {
return i
}
}
return -1
}
}
var width int
for i := 0; i < len(s); i += width {
r := rune(s[i])
if r < utf8.RuneSelf {
width = 1
} else {
r, width = utf8.DecodeRune(s[i:])
}
for _, ch := range chars {
if r == ch {
if chars == "" {
// Avoid scanning all of s.
return -1
}
if len(s) > 8 {
if as, isASCII := makeASCIISet(chars); isASCII {
for i, c := range s {
if as.contains(c) {
return i
}
}
return -1
}
}
var width int
for i := 0; i < len(s); i += width {
r := rune(s[i])
if r < utf8.RuneSelf {
width = 1
} else {
r, width = utf8.DecodeRune(s[i:])
}
for _, ch := range chars {
if r == ch {
return i
}
}
}
return -1
@ -178,25 +180,27 @@ func IndexAny(s []byte, chars string) int {
// the Unicode code points in chars. It returns -1 if chars is empty or if
// there is no code point in common.
func LastIndexAny(s []byte, chars string) int {
if len(chars) > 0 {
if len(s) > 8 {
if as, isASCII := makeASCIISet(chars); isASCII {
for i := len(s) - 1; i >= 0; i-- {
if as.contains(s[i]) {
return i
}
}
return -1
}
}
for i := len(s); i > 0; {
r, size := utf8.DecodeLastRune(s[:i])
i -= size
for _, c := range chars {
if r == c {
if chars == "" {
// Avoid scanning all of s.
return -1
}
if len(s) > 8 {
if as, isASCII := makeASCIISet(chars); isASCII {
for i := len(s) - 1; i >= 0; i-- {
if as.contains(s[i]) {
return i
}
}
return -1
}
}
for i := len(s); i > 0; {
r, size := utf8.DecodeLastRune(s[:i])
i -= size
for _, c := range chars {
if r == c {
return i
}
}
}
return -1
@ -223,7 +227,7 @@ func genSplit(s, sep []byte, sepSave, n int) [][]byte {
if m < 0 {
break
}
a[i] = s[:m+sepSave]
a[i] = s[: m+sepSave : m+sepSave]
s = s[m+len(sep):]
i++
}
@ -265,52 +269,112 @@ func SplitAfter(s, sep []byte) [][]byte {
return genSplit(s, sep, len(sep), -1)
}
// Fields splits the slice s around each instance of one or more consecutive white space
// characters, returning a slice of subslices of s or an empty list if s contains only white space.
var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1}
// Fields interprets s as a sequence of UTF-8-encoded code points.
// It splits the slice s around each instance of one or more consecutive white space
// characters, as defined by unicode.IsSpace, returning a slice of subslices of s or an
// empty slice if s contains only white space.
func Fields(s []byte) [][]byte {
return FieldsFunc(s, unicode.IsSpace)
// First count the fields.
// This is an exact count if s is ASCII, otherwise it is an approximation.
n := 0
wasSpace := 1
// setBits is used to track which bits are set in the bytes of s.
setBits := uint8(0)
for i := 0; i < len(s); i++ {
r := s[i]
setBits |= r
isSpace := int(asciiSpace[r])
n += wasSpace & ^isSpace
wasSpace = isSpace
}
if setBits >= utf8.RuneSelf {
// Some runes in the input slice are not ASCII.
return FieldsFunc(s, unicode.IsSpace)
}
// ASCII fast path
a := make([][]byte, n)
na := 0
fieldStart := 0
i := 0
// Skip spaces in the front of the input.
for i < len(s) && asciiSpace[s[i]] != 0 {
i++
}
fieldStart = i
for i < len(s) {
if asciiSpace[s[i]] == 0 {
i++
continue
}
a[na] = s[fieldStart:i:i]
na++
i++
// Skip spaces in between fields.
for i < len(s) && asciiSpace[s[i]] != 0 {
i++
}
fieldStart = i
}
if fieldStart < len(s) { // Last field might end at EOF.
a[na] = s[fieldStart:len(s):len(s)]
}
return a
}
// FieldsFunc interprets s as a sequence of UTF-8-encoded Unicode code points.
// FieldsFunc interprets s as a sequence of UTF-8-encoded code points.
// It splits the slice s at each run of code points c satisfying f(c) and
// returns a slice of subslices of s. If all code points in s satisfy f(c), or
// len(s) == 0, an empty slice is returned.
// FieldsFunc makes no guarantees about the order in which it calls f(c).
// If f does not return consistent results for a given c, FieldsFunc may crash.
func FieldsFunc(s []byte, f func(rune) bool) [][]byte {
n := 0
inField := false
// A span is used to record a slice of s of the form s[start:end].
// The start index is inclusive and the end index is exclusive.
type span struct {
start int
end int
}
spans := make([]span, 0, 32)
// Find the field start and end indices.
wasField := false
fromIndex := 0
for i := 0; i < len(s); {
r, size := utf8.DecodeRune(s[i:])
wasInField := inField
inField = !f(r)
if inField && !wasInField {
n++
size := 1
r := rune(s[i])
if r >= utf8.RuneSelf {
r, size = utf8.DecodeRune(s[i:])
}
if f(r) {
if wasField {
spans = append(spans, span{start: fromIndex, end: i})
wasField = false
}
} else {
if !wasField {
fromIndex = i
wasField = true
}
}
i += size
}
a := make([][]byte, n)
na := 0
fieldStart := -1
for i := 0; i <= len(s) && na < n; {
r, size := utf8.DecodeRune(s[i:])
if fieldStart < 0 && size > 0 && !f(r) {
fieldStart = i
i += size
continue
}
if fieldStart >= 0 && (size == 0 || f(r)) {
a[na] = s[fieldStart:i]
na++
fieldStart = -1
}
if size == 0 {
break
}
i += size
// Last field might end at EOF.
if wasField {
spans = append(spans, span{fromIndex, len(s)})
}
return a[0:na]
// Create subslices from recorded field indices.
a := make([][]byte, len(spans))
for i, span := range spans {
a[i] = s[span.start:span.end:span.end]
}
return a
}
// Join concatenates the elements of s to create a new byte slice. The separator
@ -349,8 +413,8 @@ func HasSuffix(s, suffix []byte) bool {
// Map returns a copy of the byte slice s with all its characters modified
// according to the mapping function. If mapping returns a negative value, the character is
// dropped from the string with no replacement. The characters in s and the
// output are interpreted as UTF-8-encoded Unicode code points.
// dropped from the byte slice with no replacement. The characters in s and the
// output are interpreted as UTF-8-encoded code points.
func Map(mapping func(r rune) rune, s []byte) []byte {
// In the worst case, the slice can grow when mapped, making
// things unpleasant. But it's so rare we barge in assuming it's
@ -408,28 +472,28 @@ func Repeat(b []byte, count int) []byte {
return nb
}
// ToUpper returns a copy of the byte slice s with all Unicode letters mapped to their upper case.
// ToUpper treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters within it mapped to their upper case.
func ToUpper(s []byte) []byte { return Map(unicode.ToUpper, s) }
// ToLower returns a copy of the byte slice s with all Unicode letters mapped to their lower case.
// ToLower treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their lower case.
func ToLower(s []byte) []byte { return Map(unicode.ToLower, s) }
// ToTitle returns a copy of the byte slice s with all Unicode letters mapped to their title case.
// ToTitle treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their title case.
func ToTitle(s []byte) []byte { return Map(unicode.ToTitle, s) }
// ToUpperSpecial returns a copy of the byte slice s with all Unicode letters mapped to their
// ToUpperSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their
// upper case, giving priority to the special casing rules.
func ToUpperSpecial(c unicode.SpecialCase, s []byte) []byte {
return Map(func(r rune) rune { return c.ToUpper(r) }, s)
}
// ToLowerSpecial returns a copy of the byte slice s with all Unicode letters mapped to their
// ToLowerSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their
// lower case, giving priority to the special casing rules.
func ToLowerSpecial(c unicode.SpecialCase, s []byte) []byte {
return Map(func(r rune) rune { return c.ToLower(r) }, s)
}
// ToTitleSpecial returns a copy of the byte slice s with all Unicode letters mapped to their
// ToTitleSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their
// title case, giving priority to the special casing rules.
func ToTitleSpecial(c unicode.SpecialCase, s []byte) []byte {
return Map(func(r rune) rune { return c.ToTitle(r) }, s)
@ -460,8 +524,8 @@ func isSeparator(r rune) bool {
return unicode.IsSpace(r)
}
// Title returns a copy of s with all Unicode letters that begin words
// mapped to their title case.
// Title treats s as UTF-8-encoded bytes and returns a copy with all Unicode letters that begin
// words mapped to their title case.
//
// BUG(rsc): The rule Title uses for word boundaries does not handle Unicode punctuation properly.
func Title(s []byte) []byte {
@ -481,8 +545,8 @@ func Title(s []byte) []byte {
s)
}
// TrimLeftFunc returns a subslice of s by slicing off all leading UTF-8-encoded
// Unicode code points c that satisfy f(c).
// TrimLeftFunc treats s as UTF-8-encoded bytes and returns a subslice of s by slicing off
// all leading UTF-8-encoded code points c that satisfy f(c).
func TrimLeftFunc(s []byte, f func(r rune) bool) []byte {
i := indexFunc(s, f, false)
if i == -1 {
@ -491,8 +555,8 @@ func TrimLeftFunc(s []byte, f func(r rune) bool) []byte {
return s[i:]
}
// TrimRightFunc returns a subslice of s by slicing off all trailing UTF-8
// encoded Unicode code points c that satisfy f(c).
// TrimRightFunc returns a subslice of s by slicing off all trailing
// UTF-8-encoded code points c that satisfy f(c).
func TrimRightFunc(s []byte, f func(r rune) bool) []byte {
i := lastIndexFunc(s, f, false)
if i >= 0 && s[i] >= utf8.RuneSelf {
@ -505,7 +569,7 @@ func TrimRightFunc(s []byte, f func(r rune) bool) []byte {
}
// TrimFunc returns a subslice of s by slicing off all leading and trailing
// UTF-8-encoded Unicode code points c that satisfy f(c).
// UTF-8-encoded code points c that satisfy f(c).
func TrimFunc(s []byte, f func(r rune) bool) []byte {
return TrimRightFunc(TrimLeftFunc(s, f), f)
}
@ -528,14 +592,14 @@ func TrimSuffix(s, suffix []byte) []byte {
return s
}
// IndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points.
// IndexFunc interprets s as a sequence of UTF-8-encoded code points.
// It returns the byte index in s of the first Unicode
// code point satisfying f(c), or -1 if none do.
func IndexFunc(s []byte, f func(r rune) bool) int {
return indexFunc(s, f, true)
}
// LastIndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points.
// LastIndexFunc interprets s as a sequence of UTF-8-encoded code points.
// It returns the byte index in s of the last Unicode
// code point satisfying f(c), or -1 if none do.
func LastIndexFunc(s []byte, f func(r rune) bool) int {
@ -626,19 +690,19 @@ func makeCutsetFunc(cutset string) func(r rune) bool {
}
// Trim returns a subslice of s by slicing off all leading and
// trailing UTF-8-encoded Unicode code points contained in cutset.
// trailing UTF-8-encoded code points contained in cutset.
func Trim(s []byte, cutset string) []byte {
return TrimFunc(s, makeCutsetFunc(cutset))
}
// TrimLeft returns a subslice of s by slicing off all leading
// UTF-8-encoded Unicode code points contained in cutset.
// UTF-8-encoded code points contained in cutset.
func TrimLeft(s []byte, cutset string) []byte {
return TrimLeftFunc(s, makeCutsetFunc(cutset))
}
// TrimRight returns a subslice of s by slicing off all trailing
// UTF-8-encoded Unicode code points that are contained in cutset.
// UTF-8-encoded code points that are contained in cutset.
func TrimRight(s []byte, cutset string) []byte {
return TrimRightFunc(s, makeCutsetFunc(cutset))
}
@ -649,7 +713,8 @@ func TrimSpace(s []byte) []byte {
return TrimFunc(s, unicode.IsSpace)
}
// Runes returns a slice of runes (Unicode code points) equivalent to s.
// Runes interprets s as a sequence of UTF-8-encoded code points.
// It returns a slice of runes (Unicode code points) equivalent to s.
func Runes(s []byte) []rune {
t := make([]rune, utf8.RuneCount(s))
i := 0
@ -758,3 +823,46 @@ func EqualFold(s, t []byte) bool {
// One string is empty. Are both?
return len(s) == len(t)
}
func indexRabinKarp(s, sep []byte) int {
// Rabin-Karp search
hashsep, pow := hashStr(sep)
n := len(sep)
var h uint32
for i := 0; i < n; i++ {
h = h*primeRK + uint32(s[i])
}
if h == hashsep && Equal(s[:n], sep) {
return 0
}
for i := n; i < len(s); {
h *= primeRK
h += uint32(s[i])
h -= pow * uint32(s[i-n])
i++
if h == hashsep && Equal(s[i-n:i], sep) {
return i - n
}
}
return -1
}
// primeRK is the prime base used in Rabin-Karp algorithm.
const primeRK = 16777619
// hashStr returns the hash and the appropriate multiplicative
// factor for use in Rabin-Karp algorithm.
func hashStr(sep []byte) (uint32, uint32) {
hash := uint32(0)
for i := 0; i < len(sep); i++ {
hash = hash*primeRK + uint32(sep[i])
}
var pow, sq uint32 = 1, primeRK
for i := len(sep); i > 0; i >>= 1 {
if i&1 != 0 {
pow *= sq
}
sq *= sq
}
return hash, pow
}

42
libgo/go/bytes/bytes_amd64.go
View file

@ -77,52 +77,14 @@ func Index(s, sep []byte) int {
}
return -1
}
// Rabin-Karp search
hashsep, pow := hashStr(sep)
var h uint32
for i := 0; i < n; i++ {
h = h*primeRK + uint32(s[i])
}
if h == hashsep && Equal(s[:n], sep) {
return 0
}
for i := n; i < len(s); {
h *= primeRK
h += uint32(s[i])
h -= pow * uint32(s[i-n])
i++
if h == hashsep && Equal(s[i-n:i], sep) {
return i - n
}
}
return -1
return indexRabinKarp(s, sep)
}
// Count counts the number of non-overlapping instances of sep in s.
// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s.
// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
func Count(s, sep []byte) int {
if len(sep) == 1 && cpu.X86.HasPOPCNT {
return countByte(s, sep[0])
}
return countGeneric(s, sep)
}
// primeRK is the prime base used in Rabin-Karp algorithm.
const primeRK = 16777619
// hashStr returns the hash and the appropriate multiplicative
// factor for use in Rabin-Karp algorithm.
func hashStr(sep []byte) (uint32, uint32) {
hash := uint32(0)
for i := 0; i < len(sep); i++ {
hash = hash*primeRK + uint32(sep[i])
}
var pow, sq uint32 = 1, primeRK
for i := len(sep); i > 0; i >>= 1 {
if i&1 != 0 {
pow *= sq
}
sq *= sq
}
return hash, pow
}

70
libgo/go/bytes/bytes_arm64.go Normal file
View file

@ -0,0 +1,70 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ignore
package bytes
func countByte(s []byte, c byte) int // bytes_arm64.s
// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s.
func Index(s, sep []byte) int {
n := len(sep)
switch {
case n == 0:
return 0
case n == 1:
return IndexByte(s, sep[0])
case n == len(s):
if Equal(sep, s) {
return 0
}
return -1
case n > len(s):
return -1
}
c := sep[0]
i := 0
fails := 0
t := s[:len(s)-n+1]
for i < len(t) {
if t[i] != c {
o := IndexByte(t[i:], c)
if o < 0 {
break
}
i += o
}
if Equal(s[i:i+n], sep) {
return i
}
i++
fails++
if fails >= 4+i>>4 && i < len(t) {
// Give up on IndexByte, it isn't skipping ahead
// far enough to be better than Rabin-Karp.
// Experiments (using IndexPeriodic) suggest
// the cutover is about 16 byte skips.
// TODO: if large prefixes of sep are matching
// we should cutover at even larger average skips,
// because Equal becomes that much more expensive.
// This code does not take that effect into account.
j := indexRabinKarp(s[i:], sep)
if j < 0 {
return -1
}
return i + j
}
}
return -1
}
// Count counts the number of non-overlapping instances of sep in s.
// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
func Count(s, sep []byte) int {
if len(sep) == 1 {
return countByte(s, sep[0])
}
return countGeneric(s, sep)
}

40
libgo/go/bytes/bytes_generic.go
View file

@ -2,27 +2,29 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// -build !amd64,!s390x
// -build !amd64,!s390x,!arm64
package bytes
// TODO: implements short string optimization on non amd64 platforms
// and get rid of bytes_amd64.go
// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s.
func Index(s, sep []byte) int {
n := len(sep)
if n == 0 {
switch {
case n == 0:
return 0
}
if n > len(s) {
case n == 1:
return IndexByte(s, sep[0])
case n == len(s):
if Equal(sep, s) {
return 0
}
return -1
case n > len(s):
return -1
}
c := sep[0]
if n == 1 {
return IndexByte(s, c)
}
i := 0
fails := 0
t := s[:len(s)-n+1]
for i < len(t) {
if t[i] != c {
@ -36,12 +38,28 @@ func Index(s, sep []byte) int {
return i
}
i++
fails++
if fails >= 4+i>>4 && i < len(t) {
// Give up on IndexByte, it isn't skipping ahead
// far enough to be better than Rabin-Karp.
// Experiments (using IndexPeriodic) suggest
// the cutover is about 16 byte skips.
// TODO: if large prefixes of sep are matching
// we should cutover at even larger average skips,
// because Equal becomes that much more expensive.
// This code does not take that effect into account.
j := indexRabinKarp(s[i:], sep)
if j < 0 {
return -1
}
return i + j
}
}
return -1
}
// Count counts the number of non-overlapping instances of sep in s.
// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s.
// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
func Count(s, sep []byte) int {
return countGeneric(s, sep)
}

42
libgo/go/bytes/bytes_s390x.go
View file

@ -78,49 +78,11 @@ func Index(s, sep []byte) int {
}
return -1
}
// Rabin-Karp search
hashsep, pow := hashStr(sep)
var h uint32
for i := 0; i < n; i++ {
h = h*primeRK + uint32(s[i])
}
if h == hashsep && Equal(s[:n], sep) {
return 0
}
for i := n; i < len(s); {
h *= primeRK
h += uint32(s[i])
h -= pow * uint32(s[i-n])
i++
if h == hashsep && Equal(s[i-n:i], sep) {
return i - n
}
}
return -1
return indexRabinKarp(s, sep)
}
// Count counts the number of non-overlapping instances of sep in s.
// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s.
// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
func Count(s, sep []byte) int {
return countGeneric(s, sep)
}
// primeRK is the prime base used in Rabin-Karp algorithm.
const primeRK = 16777619
// hashStr returns the hash and the appropriate multiplicative
// factor for use in Rabin-Karp algorithm.
func hashStr(sep []byte) (uint32, uint32) {
hash := uint32(0)
for i := 0; i < len(sep); i++ {
hash = hash*primeRK + uint32(sep[i])
}
var pow, sq uint32 = 1, primeRK
for i := len(sep); i > 0; i >>= 1 {
if i&1 != 0 {
pow *= sq
}
sq *= sq
}
return hash, pow
}

133
libgo/go/bytes/bytes_test.go
View file

@ -140,6 +140,9 @@ var indexTests = []BinOpTest{
{"barfoobarfooyyyzzzyyyzzzyyyzzzyyyxxxzzzyyy", "x", 33},
{"foofyfoobarfoobar", "y", 4},
{"oooooooooooooooooooooo", "r", -1},
// test fallback to Rabin-Karp.
{"oxoxoxoxoxoxoxoxoxoxoxoy", "oy", 22},
{"oxoxoxoxoxoxoxoxoxoxoxox", "oy", -1},
}
var lastIndexTests = []BinOpTest{
@ -741,6 +744,13 @@ var splittests = []SplitTest{
func TestSplit(t *testing.T) {
for _, tt := range splittests {
a := SplitN([]byte(tt.s), []byte(tt.sep), tt.n)
// Appending to the results should not change future results.
var x []byte
for _, v := range a {
x = append(v, 'z')
}
result := sliceOfString(a)
if !eq(result, tt.a) {
t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a)
@ -749,6 +759,11 @@ func TestSplit(t *testing.T) {
if tt.n == 0 {
continue
}
if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
t.Errorf("last appended result was %s; want %s", x, want)
}
s := Join(a, []byte(tt.sep))
if string(s) != tt.s {
t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s)
@ -787,11 +802,23 @@ var splitaftertests = []SplitTest{
func TestSplitAfter(t *testing.T) {
for _, tt := range splitaftertests {
a := SplitAfterN([]byte(tt.s), []byte(tt.sep), tt.n)
// Appending to the results should not change future results.
var x []byte
for _, v := range a {
x = append(v, 'z')
}
result := sliceOfString(a)
if !eq(result, tt.a) {
t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a)
continue
}
if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
t.Errorf("last appended result was %s; want %s", x, want)
}
s := Join(a, nil)
if string(s) != tt.s {
t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s)
@ -826,12 +853,29 @@ var fieldstests = []FieldsTest{
func TestFields(t *testing.T) {
for _, tt := range fieldstests {
a := Fields([]byte(tt.s))
b := []byte(tt.s)
a := Fields(b)
// Appending to the results should not change future results.
var x []byte
for _, v := range a {
x = append(v, 'z')
}
result := sliceOfString(a)
if !eq(result, tt.a) {
t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a)
continue
}
if string(b) != tt.s {
t.Errorf("slice changed to %s; want %s", string(b), tt.s)
}
if len(tt.a) > 0 {
if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
t.Errorf("last appended result was %s; want %s", x, want)
}
}
}
}
@ -852,11 +896,28 @@ func TestFieldsFunc(t *testing.T) {
{"aXXbXXXcX", []string{"a", "b", "c"}},
}
for _, tt := range fieldsFuncTests {
a := FieldsFunc([]byte(tt.s), pred)
b := []byte(tt.s)
a := FieldsFunc(b, pred)
// Appending to the results should not change future results.
var x []byte
for _, v := range a {
x = append(v, 'z')
}
result := sliceOfString(a)
if !eq(result, tt.a) {
t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a)
}
if string(b) != tt.s {
t.Errorf("slice changed to %s; want %s", b, tt.s)
}
if len(tt.a) > 0 {
if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
t.Errorf("last appended result was %s; want %s", x, want)
}
}
}
}
@ -1507,19 +1568,58 @@ var makeFieldsInput = func() []byte {
return x
}
var fieldsInput = makeFieldsInput()
var makeFieldsInputASCII = func() []byte {
x := make([]byte, 1<<20)
// Input is ~10% space, rest ASCII non-space.
for i := range x {
if rand.Intn(10) == 0 {
x[i] = ' '
} else {
x[i] = 'x'
}
}
return x
}
var bytesdata = []struct {
name string
data []byte
}{
{"ASCII", makeFieldsInputASCII()},
{"Mixed", makeFieldsInput()},
}
func BenchmarkFields(b *testing.B) {
b.SetBytes(int64(len(fieldsInput)))
for i := 0; i < b.N; i++ {
Fields(fieldsInput)
for _, sd := range bytesdata {
b.Run(sd.name, func(b *testing.B) {
for j := 1 << 4; j <= 1<<20; j <<= 4 {
b.Run(fmt.Sprintf("%d", j), func(b *testing.B) {
b.ReportAllocs()
b.SetBytes(int64(j))
data := sd.data[:j]
for i := 0; i < b.N; i++ {
Fields(data)
}
})
}
})
}
}
func BenchmarkFieldsFunc(b *testing.B) {
b.SetBytes(int64(len(fieldsInput)))
for i := 0; i < b.N; i++ {
FieldsFunc(fieldsInput, unicode.IsSpace)
for _, sd := range bytesdata {
b.Run(sd.name, func(b *testing.B) {
for j := 1 << 4; j <= 1<<20; j <<= 4 {
b.Run(fmt.Sprintf("%d", j), func(b *testing.B) {
b.ReportAllocs()
b.SetBytes(int64(j))
data := sd.data[:j]
for i := 0; i < b.N; i++ {
FieldsFunc(data, unicode.IsSpace)
}
})
}
})
}
}
@ -1638,3 +1738,18 @@ func BenchmarkTrimASCII(b *testing.B) {
}
}
}
func BenchmarkIndexPeriodic(b *testing.B) {
key := []byte{1, 1}
for _, skip := range [...]int{2, 4, 8, 16, 32, 64} {
b.Run(fmt.Sprintf("IndexPeriodic%d", skip), func(b *testing.B) {
buf := make([]byte, 1<<16)
for i := 0; i < len(buf); i += skip {
buf[i] = 1
}
for i := 0; i < b.N; i++ {
Index(buf, key)
}
})
}
}

47
libgo/go/bytes/equal_test.go
View file

@ -1,47 +0,0 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// +build linux
package bytes_test
import (
. "bytes"
"syscall"
"testing"
"unsafe"
)
// This file tests the situation where memeq is checking
// data very near to a page boundary. We want to make sure
// equal does not read across the boundary and cause a page
// fault where it shouldn't.
// This test runs only on linux. The code being tested is
// not OS-specific, so it does not need to be tested on all
// operating systems.
func TestEqualNearPageBoundary(t *testing.T) {
pagesize := syscall.Getpagesize()
b := make([]byte, 4*pagesize)
i := pagesize
for ; uintptr(unsafe.Pointer(&b[i]))%uintptr(pagesize) != 0; i++ {
}
syscall.Mprotect(b[i-pagesize:i], 0)
syscall.Mprotect(b[i+pagesize:i+2*pagesize], 0)
defer syscall.Mprotect(b[i-pagesize:i], syscall.PROT_READ|syscall.PROT_WRITE)
defer syscall.Mprotect(b[i+pagesize:i+2*pagesize], syscall.PROT_READ|syscall.PROT_WRITE)
// both of these should fault
//pagesize += int(b[i-1])
//pagesize += int(b[i+pagesize])
for j := 0; j < pagesize; j++ {
b[i+j] = 'A'
}
for j := 0; j <= pagesize; j++ {
Equal(b[i:i+j], b[i+pagesize-j:i+pagesize])
Equal(b[i+pagesize-j:i+pagesize], b[i:i+j])
}
}

141
libgo/go/bytes/example_test.go
View file

@ -119,6 +119,32 @@ func ExampleContains() {
// true
}
func ExampleContainsAny() {
fmt.Println(bytes.ContainsAny([]byte("I like seafood."), "fÄo!"))
fmt.Println(bytes.ContainsAny([]byte("I like seafood."), "去是伟大的."))
fmt.Println(bytes.ContainsAny([]byte("I like seafood."), ""))
fmt.Println(bytes.ContainsAny([]byte(""), ""))
// Output:
// true
// true
// false
// false
}
func ExampleContainsRune() {
fmt.Println(bytes.ContainsRune([]byte("I like seafood."), 'f'))
fmt.Println(bytes.ContainsRune([]byte("I like seafood."), 'ö'))
fmt.Println(bytes.ContainsRune([]byte("去是伟大的!"), '大'))
fmt.Println(bytes.ContainsRune([]byte("去是伟大的!"), '!'))
fmt.Println(bytes.ContainsRune([]byte(""), '@'))
// Output:
// true
// false
// true
// true
// false
}
func ExampleCount() {
fmt.Println(bytes.Count([]byte("cheese"), []byte("e")))
fmt.Println(bytes.Count([]byte("five"), []byte(""))) // before & after each rune
@ -127,6 +153,14 @@ func ExampleCount() {
// 5
}
func ExampleEqual() {
fmt.Println(bytes.Equal([]byte("Go"), []byte("Go")))
fmt.Println(bytes.Equal([]byte("Go"), []byte("C++")))
// Output:
// true
// false
}
func ExampleEqualFold() {
fmt.Println(bytes.EqualFold([]byte("Go"), []byte("go")))
// Output: true
@ -162,6 +196,14 @@ func ExampleIndex() {
// -1
}
func ExampleIndexByte() {
fmt.Println(bytes.IndexByte([]byte("chicken"), byte('k')))
fmt.Println(bytes.IndexByte([]byte("chicken"), byte('g')))
// Output:
// 4
// -1
}
func ExampleIndexFunc() {
f := func(c rune) bool {
return unicode.Is(unicode.Han, c)
@ -199,6 +241,36 @@ func ExampleLastIndex() {
// -1
}
func ExampleLastIndexAny() {
fmt.Println(bytes.LastIndexAny([]byte("go gopher"), "MüQp"))
fmt.Println(bytes.LastIndexAny([]byte("go 地鼠"), "地大"))
fmt.Println(bytes.LastIndexAny([]byte("go gopher"), "z,!."))
// Output:
// 5
// 3
// -1
}
func ExampleLastIndexByte() {
fmt.Println(bytes.LastIndexByte([]byte("go gopher"), byte('g')))
fmt.Println(bytes.LastIndexByte([]byte("go gopher"), byte('r')))
fmt.Println(bytes.LastIndexByte([]byte("go gopher"), byte('z')))
// Output:
// 3
// 8
// -1
}
func ExampleLastIndexFunc() {
fmt.Println(bytes.LastIndexFunc([]byte("go gopher!"), unicode.IsLetter))
fmt.Println(bytes.LastIndexFunc([]byte("go gopher!"), unicode.IsPunct))
fmt.Println(bytes.LastIndexFunc([]byte("go gopher!"), unicode.IsNumber))
// Output:
// 8
// 9
// -1
}
func ExampleJoin() {
s := [][]byte{[]byte("foo"), []byte("bar"), []byte("baz")}
fmt.Printf("%s", bytes.Join(s, []byte(", ")))
@ -218,6 +290,23 @@ func ExampleReplace() {
// moo moo moo
}
func ExampleRunes() {
rs := bytes.Runes([]byte("go gopher"))
for _, r := range rs {
fmt.Printf("%#U\n", r)
}
// Output:
// U+0067 'g'
// U+006F 'o'
// U+0020 ' '
// U+0067 'g'
// U+006F 'o'
// U+0070 'p'
// U+0068 'h'
// U+0065 'e'
// U+0072 'r'
}
func ExampleSplit() {
fmt.Printf("%q\n", bytes.Split([]byte("a,b,c"), []byte(",")))
fmt.Printf("%q\n", bytes.Split([]byte("a man a plan a canal panama"), []byte("a ")))
@ -267,6 +356,18 @@ func ExampleTrim() {
// Output: ["Achtung! Achtung"]
}
func ExampleTrimFunc() {
fmt.Println(string(bytes.TrimFunc([]byte("go-gopher!"), unicode.IsLetter)))
fmt.Println(string(bytes.TrimFunc([]byte("\"go-gopher!\""), unicode.IsLetter)))
fmt.Println(string(bytes.TrimFunc([]byte("go-gopher!"), unicode.IsPunct)))
fmt.Println(string(bytes.TrimFunc([]byte("1234go-gopher!567"), unicode.IsNumber)))
// Output:
// -gopher!
// "go-gopher!"
// go-gopher
// go-gopher!
}
func ExampleMap() {
rot13 := func(r rune) rune {
switch {
@ -281,11 +382,43 @@ func ExampleMap() {
// Output: 'Gjnf oevyyvt naq gur fyvgul tbcure...
}
func ExampleTrimLeft() {
fmt.Print(string(bytes.TrimLeft([]byte("453gopher8257"), "0123456789")))
// Output:
// gopher8257
}
func ExampleTrimLeftFunc() {
fmt.Println(string(bytes.TrimLeftFunc([]byte("go-gopher"), unicode.IsLetter)))
fmt.Println(string(bytes.TrimLeftFunc([]byte("go-gopher!"), unicode.IsPunct)))
fmt.Println(string(bytes.TrimLeftFunc([]byte("1234go-gopher!567"), unicode.IsNumber)))
// Output:
// -gopher
// go-gopher!
// go-gopher!567
}
func ExampleTrimSpace() {
fmt.Printf("%s", bytes.TrimSpace([]byte(" \t\n a lone gopher \n\t\r\n")))
// Output: a lone gopher
}
func ExampleTrimRight() {
fmt.Print(string(bytes.TrimRight([]byte("453gopher8257"), "0123456789")))
// Output:
// 453gopher
}
func ExampleTrimRightFunc() {
fmt.Println(string(bytes.TrimRightFunc([]byte("go-gopher"), unicode.IsLetter)))
fmt.Println(string(bytes.TrimRightFunc([]byte("go-gopher!"), unicode.IsPunct)))
fmt.Println(string(bytes.TrimRightFunc([]byte("1234go-gopher!567"), unicode.IsNumber)))
// Output:
// go-
// go-gopher
// 1234go-gopher!
}
func ExampleToUpper() {
fmt.Printf("%s", bytes.ToUpper([]byte("Gopher")))
// Output: GOPHER
@ -295,3 +428,11 @@ func ExampleToLower() {
fmt.Printf("%s", bytes.ToLower([]byte("Gopher")))
// Output: gopher
}
func ExampleReader_Len() {
fmt.Println(bytes.NewReader([]byte("Hi!")).Len())
fmt.Println(bytes.NewReader([]byte("こんにちは!")).Len())
// Output:
// 3
// 16
}

6
libgo/go/bytes/reader.go
View file

@ -35,6 +35,7 @@ func (r *Reader) Len() int {
// to any other method.
func (r *Reader) Size() int64 { return int64(len(r.s)) }
// Read implements the io.Reader interface.
func (r *Reader) Read(b []byte) (n int, err error) {
if r.i >= int64(len(r.s)) {
return 0, io.EOF
@ -45,6 +46,7 @@ func (r *Reader) Read(b []byte) (n int, err error) {
return
}
// ReadAt implements the io.ReaderAt interface.
func (r *Reader) ReadAt(b []byte, off int64) (n int, err error) {
// cannot modify state - see io.ReaderAt
if off < 0 {
@ -60,6 +62,7 @@ func (r *Reader) ReadAt(b []byte, off int64) (n int, err error) {
return
}
// ReadByte implements the io.ByteReader interface.
func (r *Reader) ReadByte() (byte, error) {
r.prevRune = -1
if r.i >= int64(len(r.s)) {
@ -70,6 +73,7 @@ func (r *Reader) ReadByte() (byte, error) {
return b, nil
}
// UnreadByte complements ReadByte in implementing the io.ByteScanner interface.
func (r *Reader) UnreadByte() error {
r.prevRune = -1
if r.i <= 0 {
@ -79,6 +83,7 @@ func (r *Reader) UnreadByte() error {
return nil
}
// ReadRune implements the io.RuneReader interface.
func (r *Reader) ReadRune() (ch rune, size int, err error) {
if r.i >= int64(len(r.s)) {
r.prevRune = -1
@ -94,6 +99,7 @@ func (r *Reader) ReadRune() (ch rune, size int, err error) {
return
}
// UnreadRune complements ReadRune in implementing the io.RuneScanner interface.
func (r *Reader) UnreadRune() error {
if r.prevRune < 0 {
return errors.New("bytes.Reader.UnreadRune: previous operation was not ReadRune")

4
libgo/go/bytes/reader_test.go
View file

@ -140,9 +140,9 @@ func TestReaderWriteTo(t *testing.T) {
for i := 0; i < 30; i += 3 {
var l int
if i > 0 {
l = len(data) / i
l = len(testString) / i
}
s := data[:l]
s := testString[:l]
r := NewReader(testBytes[:l])
var b Buffer
n, err := r.WriteTo(&b)

73
libgo/go/cmd/buildid/buildid.go Normal file
View file

@ -0,0 +1,73 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"flag"
"fmt"
"log"
"os"
"strings"
"cmd/internal/buildid"
)
func usage() {
fmt.Fprintf(os.Stderr, "usage: go tool buildid [-w] file\n")
flag.PrintDefaults()
os.Exit(2)
}
var wflag = flag.Bool("w", false, "write build ID")
func main() {
log.SetPrefix("buildid: ")
log.SetFlags(0)
flag.Usage = usage
flag.Parse()
if flag.NArg() != 1 {
usage()
}
file := flag.Arg(0)
id, err := buildid.ReadFile(file)
if err != nil {
log.Fatal(err)
}
if !*wflag {
fmt.Printf("%s\n", id)
return
}
f, err := os.Open(file)
if err != nil {
log.Fatal(err)
}
matches, hash, err := buildid.FindAndHash(f, id, 0)
if err != nil {
log.Fatal(err)
}
f.Close()
tail := id
if i := strings.LastIndex(id, "."); i >= 0 {
tail = tail[i+1:]
}
if len(tail) != len(hash)*2 {
log.Fatalf("%s: cannot find %d-byte hash in id %s", file, len(hash), id)
}
newID := id[:len(id)-len(tail)] + fmt.Sprintf("%x", hash)
f, err = os.OpenFile(file, os.O_WRONLY, 0)
if err != nil {
log.Fatal(err)
}
if err := buildid.Rewrite(f, matches, newID); err != nil {
log.Fatal(err)
}
if err := f.Close(); err != nil {
log.Fatal(err)
}
}

18
libgo/go/cmd/buildid/doc.go Normal file
View file

@ -0,0 +1,18 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Buildid displays or updates the build ID stored in a Go package or binary.
Usage:
go tool buildid [-w] file
By default, buildid prints the build ID found in the named file.
If the -w option is given, buildid rewrites the build ID found in
the file to accurately record a content hash of the file.
This tool is only intended for use by the go command or
other build systems.
*/
package main

272
libgo/go/cmd/cgo/ast.go
View file

@ -58,11 +58,14 @@ func (f *File) ParseGo(name string, src []byte) {
// so we use ast1 to look for the doc comments on import "C"
// and on exported functions, and we use ast2 for translating
// and reprinting.
// In cgo mode, we ignore ast2 and just apply edits directly
// the text behind ast1. In godefs mode we modify and print ast2.
ast1 := parse(name, src, parser.ParseComments)
ast2 := parse(name, src, 0)
f.Package = ast1.Name.Name
f.Name = make(map[string]*Name)
f.NamePos = make(map[*Name]token.Pos)
// In ast1, find the import "C" line and get any extra C preamble.
sawC := false
@ -96,36 +99,53 @@ func (f *File) ParseGo(name string, src []byte) {
}
// In ast2, strip the import "C" line.
w := 0
for _, decl := range ast2.Decls {
d, ok := decl.(*ast.GenDecl)
if !ok {
ast2.Decls[w] = decl
if *godefs {
w := 0
for _, decl := range ast2.Decls {
d, ok := decl.(*ast.GenDecl)
if !ok {
ast2.Decls[w] = decl
w++
continue
}
ws := 0
for _, spec := range d.Specs {
s, ok := spec.(*ast.ImportSpec)
if !ok || s.Path.Value != `"C"` {
d.Specs[ws] = spec
ws++
}
}
if ws == 0 {
continue
}
d.Specs = d.Specs[0:ws]
ast2.Decls[w] = d
w++
continue
}
ws := 0
for _, spec := range d.Specs {
s, ok := spec.(*ast.ImportSpec)
if !ok || s.Path.Value != `"C"` {
d.Specs[ws] = spec
ws++
ast2.Decls = ast2.Decls[0:w]
} else {
for _, decl := range ast2.Decls {
d, ok := decl.(*ast.GenDecl)
if !ok {
continue
}
for _, spec := range d.Specs {
if s, ok := spec.(*ast.ImportSpec); ok && s.Path.Value == `"C"` {
// Replace "C" with _ "unsafe", to keep program valid.
// (Deleting import statement or clause is not safe if it is followed
// in the source by an explicit semicolon.)
f.Edit.Replace(f.offset(s.Path.Pos()), f.offset(s.Path.End()), `_ "unsafe"`)
}
}
}
if ws == 0 {
continue
}
d.Specs = d.Specs[0:ws]
ast2.Decls[w] = d
w++
}
ast2.Decls = ast2.Decls[0:w]
// Accumulate pointers to uses of C.x.
if f.Ref == nil {
f.Ref = make([]*Ref, 0, 8)
}
f.walk(ast2, "prog", (*File).saveExprs)
f.walk(ast2, ctxProg, (*File).saveExprs)
// Accumulate exported functions.
// The comments are only on ast1 but we need to
@ -133,8 +153,8 @@ func (f *File) ParseGo(name string, src []byte) {
// The first walk fills in ExpFunc, and the
// second walk changes the entries to
// refer to ast2 instead.
f.walk(ast1, "prog", (*File).saveExport)
f.walk(ast2, "prog", (*File).saveExport2)
f.walk(ast1, ctxProg, (*File).saveExport)
f.walk(ast2, ctxProg, (*File).saveExport2)
f.Comments = ast1.Comments
f.AST = ast2
@ -143,9 +163,6 @@ func (f *File) ParseGo(name string, src []byte) {
// Like ast.CommentGroup's Text method but preserves
// leading blank lines, so that line numbers line up.
func commentText(g *ast.CommentGroup) string {
if g == nil {
return ""
}
var pieces []string
for _, com := range g.List {
c := com.Text
@ -165,7 +182,7 @@ func commentText(g *ast.CommentGroup) string {
}
// Save various references we are going to need later.
func (f *File) saveExprs(x interface{}, context string) {
func (f *File) saveExprs(x interface{}, context astContext) {
switch x := x.(type) {
case *ast.Expr:
switch (*x).(type) {
@ -178,7 +195,7 @@ func (f *File) saveExprs(x interface{}, context string) {
}
// Save references to C.xxx for later processing.
func (f *File) saveRef(n *ast.Expr, context string) {
func (f *File) saveRef(n *ast.Expr, context astContext) {
sel := (*n).(*ast.SelectorExpr)
// For now, assume that the only instance of capital C is when
// used as the imported package identifier.
@ -188,10 +205,10 @@ func (f *File) saveRef(n *ast.Expr, context string) {
if l, ok := sel.X.(*ast.Ident); !ok || l.Name != "C" {
return
}
if context == "as2" {
context = "expr"
if context == ctxAssign2 {
context = ctxExpr
}
if context == "embed-type" {
if context == ctxEmbedType {
error_(sel.Pos(), "cannot embed C type")
}
goname := sel.Sel.Name
@ -212,6 +229,7 @@ func (f *File) saveRef(n *ast.Expr, context string) {
Go: goname,
}
f.Name[goname] = name
f.NamePos[name] = sel.Pos()
}
f.Ref = append(f.Ref, &Ref{
Name: name,
@ -221,7 +239,7 @@ func (f *File) saveRef(n *ast.Expr, context string) {
}
// Save calls to C.xxx for later processing.
func (f *File) saveCall(call *ast.CallExpr, context string) {
func (f *File) saveCall(call *ast.CallExpr, context astContext) {
sel, ok := call.Fun.(*ast.SelectorExpr)
if !ok {
return
@ -229,12 +247,12 @@ func (f *File) saveCall(call *ast.CallExpr, context string) {
if l, ok := sel.X.(*ast.Ident); !ok || l.Name != "C" {
return
}
c := &Call{Call: call, Deferred: context == "defer"}
c := &Call{Call: call, Deferred: context == ctxDefer}
f.Calls = append(f.Calls, c)
}
// If a function should be exported add it to ExpFunc.
func (f *File) saveExport(x interface{}, context string) {
func (f *File) saveExport(x interface{}, context astContext) {
n, ok := x.(*ast.FuncDecl)
if !ok {
return
@ -274,7 +292,7 @@ func (f *File) saveExport(x interface{}, context string) {
}
// Make f.ExpFunc[i] point at the Func from this AST instead of the other one.
func (f *File) saveExport2(x interface{}, context string) {
func (f *File) saveExport2(x interface{}, context astContext) {
n, ok := x.(*ast.FuncDecl)
if !ok {
return
@ -288,8 +306,30 @@ func (f *File) saveExport2(x interface{}, context string) {
}
}
type astContext int
const (
ctxProg astContext = iota
ctxEmbedType
ctxType
ctxStmt
ctxExpr
ctxField
ctxParam
ctxAssign2 // assignment of a single expression to two variables
ctxSwitch
ctxTypeSwitch
ctxFile
ctxDecl
ctxSpec
ctxDefer
ctxCall // any function call other than ctxCall2
ctxCall2 // function call whose result is assigned to two variables
ctxSelector
)
// walk walks the AST x, calling visit(f, x, context) for each node.
func (f *File) walk(x interface{}, context string, visit func(*File, interface{}, string)) {
func (f *File) walk(x interface{}, context astContext, visit func(*File, interface{}, astContext)) {
visit(f, x, context)
switch n := x.(type) {
case *ast.Expr:
@ -304,10 +344,10 @@ func (f *File) walk(x interface{}, context string, visit func(*File, interface{}
// These are ordered and grouped to match ../../go/ast/ast.go
case *ast.Field:
if len(n.Names) == 0 && context == "field" {
f.walk(&n.Type, "embed-type", visit)
if len(n.Names) == 0 && context == ctxField {
f.walk(&n.Type, ctxEmbedType, visit)
} else {
f.walk(&n.Type, "type", visit)
f.walk(&n.Type, ctxType, visit)
}
case *ast.FieldList:
for _, field := range n.List {
@ -318,163 +358,163 @@ func (f *File) walk(x interface{}, context string, visit func(*File, interface{}
case *ast.Ellipsis:
case *ast.BasicLit:
case *ast.FuncLit:
f.walk(n.Type, "type", visit)
f.walk(n.Body, "stmt", visit)
f.walk(n.Type, ctxType, visit)
f.walk(n.Body, ctxStmt, visit)
case *ast.CompositeLit:
f.walk(&n.Type, "type", visit)
f.walk(n.Elts, "expr", visit)
f.walk(&n.Type, ctxType, visit)
f.walk(n.Elts, ctxExpr, visit)
case *ast.ParenExpr:
f.walk(&n.X, context, visit)
case *ast.SelectorExpr:
f.walk(&n.X, "selector", visit)
f.walk(&n.X, ctxSelector, visit)
case *ast.IndexExpr:
f.walk(&n.X, "expr", visit)
f.walk(&n.Index, "expr", visit)
f.walk(&n.X, ctxExpr, visit)
f.walk(&n.Index, ctxExpr, visit)
case *ast.SliceExpr:
f.walk(&n.X, "expr", visit)
f.walk(&n.X, ctxExpr, visit)
if n.Low != nil {
f.walk(&n.Low, "expr", visit)
f.walk(&n.Low, ctxExpr, visit)
}
if n.High != nil {
f.walk(&n.High, "expr", visit)
f.walk(&n.High, ctxExpr, visit)
}
if n.Max != nil {
f.walk(&n.Max, "expr", visit)
f.walk(&n.Max, ctxExpr, visit)
}
case *ast.TypeAssertExpr:
f.walk(&n.X, "expr", visit)
f.walk(&n.Type, "type", visit)
f.walk(&n.X, ctxExpr, visit)
f.walk(&n.Type, ctxType, visit)
case *ast.CallExpr:
if context == "as2" {
f.walk(&n.Fun, "call2", visit)
if context == ctxAssign2 {
f.walk(&n.Fun, ctxCall2, visit)
} else {
f.walk(&n.Fun, "call", visit)
f.walk(&n.Fun, ctxCall, visit)
}
f.walk(n.Args, "expr", visit)
f.walk(n.Args, ctxExpr, visit)
case *ast.StarExpr:
f.walk(&n.X, context, visit)
case *ast.UnaryExpr:
f.walk(&n.X, "expr", visit)
f.walk(&n.X, ctxExpr, visit)
case *ast.BinaryExpr:
f.walk(&n.X, "expr", visit)
f.walk(&n.Y, "expr", visit)
f.walk(&n.X, ctxExpr, visit)
f.walk(&n.Y, ctxExpr, visit)
case *ast.KeyValueExpr:
f.walk(&n.Key, "expr", visit)
f.walk(&n.Value, "expr", visit)
f.walk(&n.Key, ctxExpr, visit)
f.walk(&n.Value, ctxExpr, visit)
case *ast.ArrayType:
f.walk(&n.Len, "expr", visit)
f.walk(&n.Elt, "type", visit)
f.walk(&n.Len, ctxExpr, visit)
f.walk(&n.Elt, ctxType, visit)
case *ast.StructType:
f.walk(n.Fields, "field", visit)
f.walk(n.Fields, ctxField, visit)
case *ast.FuncType:
f.walk(n.Params, "param", visit)
f.walk(n.Params, ctxParam, visit)
if n.Results != nil {
f.walk(n.Results, "param", visit)
f.walk(n.Results, ctxParam, visit)
}
case *ast.InterfaceType:
f.walk(n.Methods, "field", visit)
f.walk(n.Methods, ctxField, visit)
case *ast.MapType:
f.walk(&n.Key, "type", visit)
f.walk(&n.Value, "type", visit)
f.walk(&n.Key, ctxType, visit)
f.walk(&n.Value, ctxType, visit)
case *ast.ChanType:
f.walk(&n.Value, "type", visit)
f.walk(&n.Value, ctxType, visit)
case *ast.BadStmt:
case *ast.DeclStmt:
f.walk(n.Decl, "decl", visit)
f.walk(n.Decl, ctxDecl, visit)
case *ast.EmptyStmt:
case *ast.LabeledStmt:
f.walk(n.Stmt, "stmt", visit)
f.walk(n.Stmt, ctxStmt, visit)
case *ast.ExprStmt:
f.walk(&n.X, "expr", visit)
f.walk(&n.X, ctxExpr, visit)
case *ast.SendStmt:
f.walk(&n.Chan, "expr", visit)
f.walk(&n.Value, "expr", visit)
f.walk(&n.Chan, ctxExpr, visit)
f.walk(&n.Value, ctxExpr, visit)
case *ast.IncDecStmt:
f.walk(&n.X, "expr", visit)
f.walk(&n.X, ctxExpr, visit)
case *ast.AssignStmt:
f.walk(n.Lhs, "expr", visit)
f.walk(n.Lhs, ctxExpr, visit)
if len(n.Lhs) == 2 && len(n.Rhs) == 1 {
f.walk(n.Rhs, "as2", visit)
f.walk(n.Rhs, ctxAssign2, visit)
} else {
f.walk(n.Rhs, "expr", visit)
f.walk(n.Rhs, ctxExpr, visit)
}
case *ast.GoStmt:
f.walk(n.Call, "expr", visit)
f.walk(n.Call, ctxExpr, visit)
case *ast.DeferStmt:
f.walk(n.Call, "defer", visit)
f.walk(n.Call, ctxDefer, visit)
case *ast.ReturnStmt:
f.walk(n.Results, "expr", visit)
f.walk(n.Results, ctxExpr, visit)
case *ast.BranchStmt:
case *ast.BlockStmt:
f.walk(n.List, context, visit)
case *ast.IfStmt:
f.walk(n.Init, "stmt", visit)
f.walk(&n.Cond, "expr", visit)
f.walk(n.Body, "stmt", visit)
f.walk(n.Else, "stmt", visit)
f.walk(n.Init, ctxStmt, visit)
f.walk(&n.Cond, ctxExpr, visit)
f.walk(n.Body, ctxStmt, visit)
f.walk(n.Else, ctxStmt, visit)
case *ast.CaseClause:
if context == "typeswitch" {
context = "type"
if context == ctxTypeSwitch {
context = ctxType
} else {
context = "expr"
context = ctxExpr
}
f.walk(n.List, context, visit)
f.walk(n.Body, "stmt", visit)
f.walk(n.Body, ctxStmt, visit)
case *ast.SwitchStmt:
f.walk(n.Init, "stmt", visit)
f.walk(&n.Tag, "expr", visit)
f.walk(n.Body, "switch", visit)
f.walk(n.Init, ctxStmt, visit)
f.walk(&n.Tag, ctxExpr, visit)
f.walk(n.Body, ctxSwitch, visit)
case *ast.TypeSwitchStmt:
f.walk(n.Init, "stmt", visit)
f.walk(n.Assign, "stmt", visit)
f.walk(n.Body, "typeswitch", visit)
f.walk(n.Init, ctxStmt, visit)
f.walk(n.Assign, ctxStmt, visit)
f.walk(n.Body, ctxTypeSwitch, visit)
case *ast.CommClause:
f.walk(n.Comm, "stmt", visit)
f.walk(n.Body, "stmt", visit)
f.walk(n.Comm, ctxStmt, visit)
f.walk(n.Body, ctxStmt, visit)
case *ast.SelectStmt:
f.walk(n.Body, "stmt", visit)
f.walk(n.Body, ctxStmt, visit)
case *ast.ForStmt:
f.walk(n.Init, "stmt", visit)
f.walk(&n.Cond, "expr", visit)
f.walk(n.Post, "stmt", visit)
f.walk(n.Body, "stmt", visit)
f.walk(n.Init, ctxStmt, visit)
f.walk(&n.Cond, ctxExpr, visit)
f.walk(n.Post, ctxStmt, visit)
f.walk(n.Body, ctxStmt, visit)
case *ast.RangeStmt:
f.walk(&n.Key, "expr", visit)
f.walk(&n.Value, "expr", visit)
f.walk(&n.X, "expr", visit)
f.walk(n.Body, "stmt", visit)
f.walk(&n.Key, ctxExpr, visit)
f.walk(&n.Value, ctxExpr, visit)
f.walk(&n.X, ctxExpr, visit)
f.walk(n.Body, ctxStmt, visit)
case *ast.ImportSpec:
case *ast.ValueSpec:
f.walk(&n.Type, "type", visit)
f.walk(&n.Type, ctxType, visit)
if len(n.Names) == 2 && len(n.Values) == 1 {
f.walk(&n.Values[0], "as2", visit)
f.walk(&n.Values[0], ctxAssign2, visit)
} else {
f.walk(n.Values, "expr", visit)
f.walk(n.Values, ctxExpr, visit)
}
case *ast.TypeSpec:
f.walk(&n.Type, "type", visit)
f.walk(&n.Type, ctxType, visit)
case *ast.BadDecl:
case *ast.GenDecl:
f.walk(n.Specs, "spec", visit)
f.walk(n.Specs, ctxSpec, visit)
case *ast.FuncDecl:
if n.Recv != nil {
f.walk(n.Recv, "param", visit)
f.walk(n.Recv, ctxParam, visit)
}
f.walk(n.Type, "type", visit)
f.walk(n.Type, ctxType, visit)
if n.Body != nil {
f.walk(n.Body, "stmt", visit)
f.walk(n.Body, ctxStmt, visit)
}
case *ast.File:
f.walk(n.Decls, "decl", visit)
f.walk(n.Decls, ctxDecl, visit)
case *ast.Package:
for _, file := range n.Files {
f.walk(file, "file", visit)
f.walk(file, ctxFile, visit)
}
case []ast.Decl:

281
libgo/go/cmd/cgo/doc.go
View file

@ -102,11 +102,13 @@ the use of cgo, and to 0 to disable it. The go tool will set the
build constraint "cgo" if cgo is enabled.
When cross-compiling, you must specify a C cross-compiler for cgo to
use. You can do this by setting the CC_FOR_TARGET environment
variable when building the toolchain using make.bash, or by setting
the CC environment variable any time you run the go tool. The
CXX_FOR_TARGET and CXX environment variables work in a similar way for
C++ code.
use. You can do this by setting the generic CC_FOR_TARGET or the
more specific CC_FOR_${GOOS}_${GOARCH} (for example, CC_FOR_linux_arm)
environment variable when building the toolchain using make.bash,
or you can set the CC environment variable any time you run the go tool.
The CXX_FOR_TARGET, CXX_FOR_${GOOS}_${GOARCH}, and CXX
environment variables work in a similar way for C++ code.
Go references to C
@ -126,12 +128,29 @@ C.complexfloat (complex float), and C.complexdouble (complex double).
The C type void* is represented by Go's unsafe.Pointer.
The C types __int128_t and __uint128_t are represented by [16]byte.
A few special C types which would normally be represented by a pointer
type in Go are instead represented by a uintptr. See the Special
cases section below.
To access a struct, union, or enum type directly, prefix it with
struct_, union_, or enum_, as in C.struct_stat.
The size of any C type T is available as C.sizeof_T, as in
C.sizeof_struct_stat.
A C function may be declared in the Go file with a parameter type of
the special name _GoString_. This function may be called with an
ordinary Go string value. The string length, and a pointer to the
string contents, may be accessed by calling the C functions
size_t _GoStringLen(_GoString_ s);
const char *_GoStringPtr(_GoString_ s);
These functions are only available in the preamble, not in other C
files. The C code must not modify the contents of the pointer returned
by _GoStringPtr. Note that the string contents may not have a trailing
NUL byte.
As Go doesn't have support for C's union type in the general case,
C's union types are represented as a Go byte array with the same length.
@ -241,7 +260,16 @@ They will be available in the C code as:
found in the _cgo_export.h generated header, after any preambles
copied from the cgo input files. Functions with multiple
return values are mapped to functions returning a struct.
Not all Go types can be mapped to C types in a useful way.
Go struct types are not supported; use a C struct type.
Go array types are not supported; use a C pointer.
Go functions that take arguments of type string may be called with the
C type _GoString_, described above. The _GoString_ type will be
automatically defined in the preamble. Note that there is no way for C
code to create a value of this type; this is only useful for passing
string values from Go to C and back to Go.
Using //export in a file places a restriction on the preamble:
since it is copied into two different C output files, it must not
@ -264,6 +292,14 @@ pointer is a Go pointer or a C pointer is a dynamic property
determined by how the memory was allocated; it has nothing to do with
the type of the pointer.
Note that values of some Go types, other than the type's zero value,
always include Go pointers. This is true of string, slice, interface,
channel, map, and function types. A pointer type may hold a Go pointer
or a C pointer. Array and struct types may or may not include Go
pointers, depending on the element types. All the discussion below
about Go pointers applies not just to pointer types, but also to other
types that include Go pointers.
Go code may pass a Go pointer to C provided the Go memory to which it
points does not contain any Go pointers. The C code must preserve
this property: it must not store any Go pointers in Go memory, even
@ -274,14 +310,17 @@ the Go memory in question is the entire array or the entire backing
array of the slice.
C code may not keep a copy of a Go pointer after the call returns.
This includes the _GoString_ type, which, as noted above, includes a
Go pointer; _GoString_ values may not be retained by C code.
A Go function called by C code may not return a Go pointer. A Go
function called by C code may take C pointers as arguments, and it may
store non-pointer or C pointer data through those pointers, but it may
not store a Go pointer in memory pointed to by a C pointer. A Go
function called by C code may take a Go pointer as an argument, but it
must preserve the property that the Go memory to which it points does
not contain any Go pointers.
A Go function called by C code may not return a Go pointer (which
implies that it may not return a string, slice, channel, and so
forth). A Go function called by C code may take C pointers as
arguments, and it may store non-pointer or C pointer data through
those pointers, but it may not store a Go pointer in memory pointed to
by a C pointer. A Go function called by C code may take a Go pointer
as an argument, but it must preserve the property that the Go memory
to which it points does not contain any Go pointers.
Go code may not store a Go pointer in C memory. C code may store Go
pointers in C memory, subject to the rule above: it must stop storing
@ -299,6 +338,84 @@ and of course there is nothing stopping the C code from doing anything
it likes. However, programs that break these rules are likely to fail
in unexpected and unpredictable ways.
Special cases
A few special C types which would normally be represented by a pointer
type in Go are instead represented by a uintptr. Those types are
the CF*Ref types from the CoreFoundation library on Darwin, including:
CFAllocatorRef
CFArrayRef
CFAttributedStringRef
CFBagRef
CFBinaryHeapRef
CFBitVectorRef
CFBooleanRef
CFBundleRef
CFCalendarRef
CFCharacterSetRef
CFDataRef
CFDateFormatterRef
CFDateRef
CFDictionaryRef
CFErrorRef
CFFileDescriptorRef
CFFileSecurityRef
CFLocaleRef
CFMachPortRef
CFMessagePortRef
CFMutableArrayRef
CFMutableAttributedStringRef
CFMutableBagRef
CFMutableBitVectorRef
CFMutableCharacterSetRef
CFMutableDataRef
CFMutableDictionaryRef
CFMutableSetRef
CFMutableStringRef
CFNotificationCenterRef
CFNullRef
CFNumberFormatterRef
CFNumberRef
CFPlugInInstanceRef
CFPlugInRef
CFPropertyListRef
CFReadStreamRef
CFRunLoopObserverRef
CFRunLoopRef
CFRunLoopSourceRef
CFRunLoopTimerRef
CFSetRef
CFSocketRef
CFStringRef
CFStringTokenizerRef
CFTimeZoneRef
CFTreeRef
CFTypeRef
CFURLCreateFromFSRef
CFURLEnumeratorRef
CFURLGetFSRef
CFURLRef
CFUUIDRef
CFUserNotificationRef
CFWriteStreamRef
CFXMLNodeRef
CFXMLParserRef
CFXMLTreeRef
These types are uintptr on the Go side because they would otherwise
confuse the Go garbage collector; they are sometimes not really
pointers but data structures encoded in a pointer type. All operations
on these types must happen in C. The proper constant to initialize an
empty such reference is 0, not nil.
This special case was introduced in Go 1.10. For auto-updating code
from Go 1.9 and earlier, use the cftype rewrite in the Go fix tool:
go tool fix -r cftype <pkg>
It will replace nil with 0 in the appropriate places.
Using cgo directly
Usage:
@ -312,32 +429,35 @@ invoking the C compiler to compile the C parts of the package.
The following options are available when running cgo directly:
-V
Print cgo version and exit.
-debug-define
Debugging option. Print #defines.
-debug-gcc
Debugging option. Trace C compiler execution and output.
-dynimport file
Write list of symbols imported by file. Write to
-dynout argument or to standard output. Used by go
build when building a cgo package.
-dynlinker
Write dynamic linker as part of -dynimport output.
-dynout file
Write -dynimport output to file.
-dynpackage package
Set Go package for -dynimport output.
-dynlinker
Write dynamic linker as part of -dynimport output.
-godefs
Write out input file in Go syntax replacing C package
names with real values. Used to generate files in the
syscall package when bootstrapping a new target.
-srcdir directory
Find the Go input files, listed on the command line,
in directory.
-objdir directory
Put all generated files in directory.
-importpath string
The import path for the Go package. Optional; used for
nicer comments in the generated files.
-exportheader file
If there are any exported functions, write the
generated export declarations to file.
C code can #include this to see the declarations.
-importpath string
The import path for the Go package. Optional; used for
nicer comments in the generated files.
-import_runtime_cgo
If set (which it is by default) import runtime/cgo in
generated output.
-import_syscall
If set (which it is by default) import syscall in
generated output.
-gccgo
Generate output for the gccgo compiler rather than the
gc compiler.
@ -345,16 +465,13 @@ The following options are available when running cgo directly:
The -fgo-prefix option to be used with gccgo.
-gccgopkgpath path
The -fgo-pkgpath option to be used with gccgo.
-import_runtime_cgo
If set (which it is by default) import runtime/cgo in
generated output.
-import_syscall
If set (which it is by default) import syscall in
generated output.
-debug-define
Debugging option. Print #defines.
-debug-gcc
Debugging option. Trace C compiler execution and output.
-godefs
Write out input file in Go syntax replacing C package
names with real values. Used to generate files in the
syscall package when bootstrapping a new target.
-objdir directory
Put all generated files in directory.
-srcdir directory
*/
package main
@ -403,21 +520,19 @@ about simple #defines for constants and the like. These are recorded
for later use.
Next, cgo needs to identify the kinds for each identifier. For the
identifiers C.foo and C.bar, cgo generates this C program:
identifiers C.foo, cgo generates this C program:
<preamble>
#line 1 "not-declared"
void __cgo_f_xxx_1(void) { __typeof__(foo) *__cgo_undefined__; }
void __cgo_f_1_1(void) { __typeof__(foo) *__cgo_undefined__1; }
#line 1 "not-type"
void __cgo_f_xxx_2(void) { foo *__cgo_undefined__; }
#line 1 "not-const"
void __cgo_f_xxx_3(void) { enum { __cgo_undefined__ = (foo)*1 }; }
#line 2 "not-declared"
void __cgo_f_xxx_1(void) { __typeof__(bar) *__cgo_undefined__; }
#line 2 "not-type"
void __cgo_f_xxx_2(void) { bar *__cgo_undefined__; }
#line 2 "not-const"
void __cgo_f_xxx_3(void) { enum { __cgo_undefined__ = (bar)*1 }; }
void __cgo_f_1_2(void) { foo *__cgo_undefined__2; }
#line 1 "not-int-const"
void __cgo_f_1_3(void) { enum { __cgo_undefined__3 = (foo)*1 }; }
#line 1 "not-num-const"
void __cgo_f_1_4(void) { static const double __cgo_undefined__4 = (foo); }
#line 1 "not-str-lit"
void __cgo_f_1_5(void) { static const char __cgo_undefined__5[] = (foo); }
This program will not compile, but cgo can use the presence or absence
of an error message on a given line to deduce the information it
@ -427,45 +542,72 @@ errors that might stop parsing early.
An error on not-declared:1 indicates that foo is undeclared.
An error on not-type:1 indicates that foo is not a type (if declared at all, it is an identifier).
An error on not-const:1 indicates that foo is not an integer constant.
An error on not-int-const:1 indicates that foo is not an integer constant.
An error on not-num-const:1 indicates that foo is not a number constant.
An error on not-str-lit:1 indicates that foo is not a string literal.
An error on not-signed-int-const:1 indicates that foo is not a signed integer constant.
The line number specifies the name involved. In the example, 1 is foo and 2 is bar.
The line number specifies the name involved. In the example, 1 is foo.
Next, cgo must learn the details of each type, variable, function, or
constant. It can do this by reading object files. If cgo has decided
that t1 is a type, v2 and v3 are variables or functions, and c4, c5,
and c6 are constants, it generates:
that t1 is a type, v2 and v3 are variables or functions, and i4, i5
are integer constants, u6 is an unsigned integer constant, and f7 and f8
are float constants, and s9 and s10 are string constants, it generates:
<preamble>
__typeof__(t1) *__cgo__1;
__typeof__(v2) *__cgo__2;
__typeof__(v3) *__cgo__3;
__typeof__(c4) *__cgo__4;
enum { __cgo_enum__4 = c4 };
__typeof__(c5) *__cgo__5;
enum { __cgo_enum__5 = c5 };
__typeof__(c6) *__cgo__6;
enum { __cgo_enum__6 = c6 };
__typeof__(i4) *__cgo__4;
enum { __cgo_enum__4 = i4 };
__typeof__(i5) *__cgo__5;
enum { __cgo_enum__5 = i5 };
__typeof__(u6) *__cgo__6;
enum { __cgo_enum__6 = u6 };
__typeof__(f7) *__cgo__7;
__typeof__(f8) *__cgo__8;
__typeof__(s9) *__cgo__9;
__typeof__(s10) *__cgo__10;
long long __cgo_debug_data[] = {
long long __cgodebug_ints[] = {
0, // t1
0, // v2
0, // v3
c4,
c5,
c6,
i4,
i5,
u6,
0, // f7
0, // f8
0, // s9
0, // s10
1
};
double __cgodebug_floats[] = {
0, // t1
0, // v2
0, // v3
0, // i4
0, // i5
0, // u6
f7,
f8,
0, // s9
0, // s10
1
};
const char __cgodebug_str__9[] = s9;
const unsigned long long __cgodebug_strlen__9 = sizeof(s9)-1;
const char __cgodebug_str__10[] = s10;
const unsigned long long __cgodebug_strlen__10 = sizeof(s10)-1;
and again invokes the system C compiler, to produce an object file
containing debug information. Cgo parses the DWARF debug information
for __cgo__N to learn the type of each identifier. (The types also
distinguish functions from global variables.) If using a standard gcc,
cgo can parse the DWARF debug information for the __cgo_enum__N to
learn the identifier's value. The LLVM-based gcc on OS X emits
incomplete DWARF information for enums; in that case cgo reads the
constant values from the __cgo_debug_data from the object file's data
segment.
distinguish functions from global variables.) Cgo reads the constant
values from the __cgodebug_* from the object file's data segment.
At this point cgo knows the meaning of each C.xxx well enough to start
the translation process.
@ -550,9 +692,12 @@ _cgo_main.c:
int main() { return 0; }
void crosscall2(void(*fn)(void*, int, uintptr_t), void *a, int c, uintptr_t ctxt) { }
uintptr_t _cgo_wait_runtime_init_done() { }
uintptr_t _cgo_wait_runtime_init_done() { return 0; }
void _cgo_release_context(uintptr_t ctxt) { }
char* _cgo_topofstack(void) { return (char*)0; }
void _cgo_allocate(void *a, int c) { }
void _cgo_panic(void *a, int c) { }
void _cgo_reginit(void) { }
The extra functions here are stubs to satisfy the references in the C
code generated for gcc. The build process links this stub, along with

277
libgo/go/cmd/cgo/gcc.go
View file

@ -188,21 +188,8 @@ func (p *Package) Translate(f *File) {
p.loadDWARF(f, needType)
}
if p.rewriteCalls(f) {
// Add `import _cgo_unsafe "unsafe"` as the first decl
// after the package statement.
imp := &ast.GenDecl{
Tok: token.IMPORT,
Specs: []ast.Spec{
&ast.ImportSpec{
Name: ast.NewIdent("_cgo_unsafe"),
Path: &ast.BasicLit{
Kind: token.STRING,
Value: `"unsafe"`,
},
},
},
}
f.AST.Decls = append([]ast.Decl{imp}, f.AST.Decls...)
// Add `import _cgo_unsafe "unsafe"` after the package statement.
f.Edit.Insert(f.offset(f.AST.Name.End()), "; import _cgo_unsafe \"unsafe\"")
}
p.rewriteRef(f)
}
@ -211,8 +198,8 @@ func (p *Package) Translate(f *File) {
// in the file f and saves relevant renamings in f.Name[name].Define.
func (p *Package) loadDefines(f *File) {
var b bytes.Buffer
b.WriteString(f.Preamble)
b.WriteString(builtinProlog)
b.WriteString(f.Preamble)
stdout := p.gccDefines(b.Bytes())
for _, line := range strings.Split(stdout, "\n") {
@ -283,10 +270,6 @@ func (p *Package) guessKinds(f *File) []*Name {
if n.IsConst() {
continue
}
if isName(n.Define) {
n.C = n.Define
}
}
// If this is a struct, union, or enum type name, no need to guess the kind.
@ -315,57 +298,45 @@ func (p *Package) guessKinds(f *File) []*Name {
// For each name, we generate these lines, where xxx is the index in toSniff plus one.
//
// #line xxx "not-declared"
// void __cgo_f_xxx_1(void) { __typeof__(name) *__cgo_undefined__; }
// void __cgo_f_xxx_1(void) { __typeof__(name) *__cgo_undefined__1; }
// #line xxx "not-type"
// void __cgo_f_xxx_2(void) { name *__cgo_undefined__; }
// void __cgo_f_xxx_2(void) { name *__cgo_undefined__2; }
// #line xxx "not-int-const"
// void __cgo_f_xxx_3(void) { enum { __cgo_undefined__ = (name)*1 }; }
// void __cgo_f_xxx_3(void) { enum { __cgo_undefined__3 = (name)*1 }; }
// #line xxx "not-num-const"
// void __cgo_f_xxx_4(void) { static const double x = (name); }
// void __cgo_f_xxx_4(void) { static const double __cgo_undefined__4 = (name); }
// #line xxx "not-str-lit"
// void __cgo_f_xxx_5(void) { static const char x[] = (name); }
// #line xxx "not-signed-int-const"
// #if 0 < -(name)
// #line xxx "not-signed-int-const"
// #error found unsigned int
// #endif
// void __cgo_f_xxx_5(void) { static const char __cgo_undefined__5[] = (name); }
//
// If we see an error at not-declared:xxx, the corresponding name is not declared.
// If we see an error at not-type:xxx, the corresponding name is a type.
// If we see an error at not-int-const:xxx, the corresponding name is not an integer constant.
// If we see an error at not-num-const:xxx, the corresponding name is not a number constant.
// If we see an error at not-str-lit:xxx, the corresponding name is not a string literal.
// If we see an error at not-signed-int-const:xxx, the corresponding name is not a signed integer literal.
//
// The specific input forms are chosen so that they are valid C syntax regardless of
// whether name denotes a type or an expression.
var b bytes.Buffer
b.WriteString(f.Preamble)
b.WriteString(builtinProlog)
b.WriteString(f.Preamble)
for i, n := range names {
fmt.Fprintf(&b, "#line %d \"not-declared\"\n"+
"void __cgo_f_%d_1(void) { __typeof__(%s) *__cgo_undefined__; }\n"+
"void __cgo_f_%d_1(void) { __typeof__(%s) *__cgo_undefined__1; }\n"+
"#line %d \"not-type\"\n"+
"void __cgo_f_%d_2(void) { %s *__cgo_undefined__; }\n"+
"void __cgo_f_%d_2(void) { %s *__cgo_undefined__2; }\n"+
"#line %d \"not-int-const\"\n"+
"void __cgo_f_%d_3(void) { enum { __cgo_undefined__ = (%s)*1 }; }\n"+
"void __cgo_f_%d_3(void) { enum { __cgo_undefined__3 = (%s)*1 }; }\n"+
"#line %d \"not-num-const\"\n"+
"void __cgo_f_%d_4(void) { static const double x = (%s); }\n"+
"void __cgo_f_%d_4(void) { static const double __cgo_undefined__4 = (%s); }\n"+
"#line %d \"not-str-lit\"\n"+
"void __cgo_f_%d_5(void) { static const char s[] = (%s); }\n"+
"#line %d \"not-signed-int-const\"\n"+
"#if 0 < (%s)\n"+
"#line %d \"not-signed-int-const\"\n"+
"#error found unsigned int\n"+
"#endif\n",
"void __cgo_f_%d_5(void) { static const char __cgo_undefined__5[] = (%s); }\n",
i+1, i+1, n.C,
i+1, i+1, n.C,
i+1, i+1, n.C,
i+1, i+1, n.C,
i+1, i+1, n.C,
i+1, n.C, i+1,
)
}
fmt.Fprintf(&b, "#line 1 \"completed\"\n"+
@ -384,7 +355,6 @@ func (p *Package) guessKinds(f *File) []*Name {
notNumConst
notStrLiteral
notDeclared
notSignedIntConst
)
sawUnmatchedErrors := false
for _, line := range strings.Split(stderr, "\n") {
@ -438,8 +408,6 @@ func (p *Package) guessKinds(f *File) []*Name {
sniff[i] |= notNumConst
case "not-str-lit":
sniff[i] |= notStrLiteral
case "not-signed-int-const":
sniff[i] |= notSignedIntConst
default:
if isError {
sawUnmatchedErrors = true
@ -455,22 +423,11 @@ func (p *Package) guessKinds(f *File) []*Name {
}
for i, n := range names {
switch sniff[i] &^ notSignedIntConst {
switch sniff[i] {
default:
var tpos token.Pos
for _, ref := range f.Ref {
if ref.Name == n {
tpos = ref.Pos()
break
}
}
error_(tpos, "could not determine kind of name for C.%s", fixGo(n.Go))
error_(f.NamePos[n], "could not determine kind of name for C.%s", fixGo(n.Go))
case notStrLiteral | notType:
if sniff[i]&notSignedIntConst != 0 {
n.Kind = "uconst"
} else {
n.Kind = "iconst"
}
n.Kind = "iconst"
case notIntConst | notStrLiteral | notType:
n.Kind = "fconst"
case notIntConst | notNumConst | notType:
@ -510,12 +467,12 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
// for each entry in names and then dereference the type we
// learn for __cgo__i.
var b bytes.Buffer
b.WriteString(f.Preamble)
b.WriteString(builtinProlog)
b.WriteString(f.Preamble)
b.WriteString("#line 1 \"cgo-dwarf-inference\"\n")
for i, n := range names {
fmt.Fprintf(&b, "__typeof__(%s) *__cgo__%d;\n", n.C, i)
if n.Kind == "iconst" || n.Kind == "uconst" {
if n.Kind == "iconst" {
fmt.Fprintf(&b, "enum { __cgo_enum__%d = %s };\n", i, n.C)
}
}
@ -524,7 +481,7 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
// so we can read them out of the object file.
fmt.Fprintf(&b, "long long __cgodebug_ints[] = {\n")
for _, n := range names {
if n.Kind == "iconst" || n.Kind == "uconst" {
if n.Kind == "iconst" {
fmt.Fprintf(&b, "\t%s,\n", n.C)
} else {
fmt.Fprintf(&b, "\t0,\n")
@ -562,14 +519,6 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
// Scan DWARF info for top-level TagVariable entries with AttrName __cgo__i.
types := make([]dwarf.Type, len(names))
nameToIndex := make(map[*Name]int)
for i, n := range names {
nameToIndex[n] = i
}
nameToRef := make(map[*Name]*Ref)
for _, ref := range f.Ref {
nameToRef[ref.Name] = ref
}
r := d.Reader()
for {
e, err := r.Next()
@ -620,10 +569,7 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
if types[i] == nil {
continue
}
pos := token.NoPos
if ref, ok := nameToRef[n]; ok {
pos = ref.Pos()
}
pos := f.NamePos[n]
f, fok := types[i].(*dwarf.FuncType)
if n.Kind != "type" && fok {
n.Kind = "func"
@ -633,11 +579,11 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
switch n.Kind {
case "iconst":
if i < len(ints) {
n.Const = fmt.Sprintf("%#x", ints[i])
}
case "uconst":
if i < len(ints) {
n.Const = fmt.Sprintf("%#x", uint64(ints[i]))
if _, ok := types[i].(*dwarf.UintType); ok {
n.Const = fmt.Sprintf("%#x", uint64(ints[i]))
} else {
n.Const = fmt.Sprintf("%#x", ints[i])
}
}
case "fconst":
if i < len(floats) {
@ -778,8 +724,9 @@ func (p *Package) rewriteCall(f *File, call *Call, name *Name) bool {
stmts = append(stmts, stmt)
}
const cgoMarker = "__cgo__###__marker__"
fcall := &ast.CallExpr{
Fun: call.Call.Fun,
Fun: ast.NewIdent(cgoMarker),
Args: nargs,
}
ftype := &ast.FuncType{
@ -801,31 +748,26 @@ func (p *Package) rewriteCall(f *File, call *Call, name *Name) bool {
}
}
// There is a Ref pointing to the old call.Call.Fun.
// If this call expects two results, we have to
// adjust the results of the function we generated.
for _, ref := range f.Ref {
if ref.Expr == &call.Call.Fun {
ref.Expr = &fcall.Fun
// If this call expects two results, we have to
// adjust the results of the function we generated.
if ref.Context == "call2" {
if ftype.Results == nil {
// An explicit void argument
// looks odd but it seems to
// be how cgo has worked historically.
ftype.Results = &ast.FieldList{
List: []*ast.Field{
&ast.Field{
Type: ast.NewIdent("_Ctype_void"),
},
if ref.Expr == &call.Call.Fun && ref.Context == ctxCall2 {
if ftype.Results == nil {
// An explicit void argument
// looks odd but it seems to
// be how cgo has worked historically.
ftype.Results = &ast.FieldList{
List: []*ast.Field{
&ast.Field{
Type: ast.NewIdent("_Ctype_void"),
},
}
},
}
ftype.Results.List = append(ftype.Results.List,
&ast.Field{
Type: ast.NewIdent("error"),
})
}
ftype.Results.List = append(ftype.Results.List,
&ast.Field{
Type: ast.NewIdent("error"),
})
}
}
@ -839,14 +781,16 @@ func (p *Package) rewriteCall(f *File, call *Call, name *Name) bool {
Results: []ast.Expr{fcall},
}
}
call.Call.Fun = &ast.FuncLit{
lit := &ast.FuncLit{
Type: ftype,
Body: &ast.BlockStmt{
List: append(stmts, fbody),
},
}
call.Call.Lparen = token.NoPos
call.Call.Rparen = token.NoPos
text := strings.Replace(gofmt(lit), "\n", ";", -1)
repl := strings.Split(text, cgoMarker)
f.Edit.Insert(f.offset(call.Call.Fun.Pos()), repl[0])
f.Edit.Insert(f.offset(call.Call.Fun.End()), repl[1])
return needsUnsafe
}
@ -1000,8 +944,8 @@ func (p *Package) checkAddrArgs(f *File, args []ast.Expr, x ast.Expr) []ast.Expr
// effect is a function call.
func (p *Package) hasSideEffects(f *File, x ast.Expr) bool {
found := false
f.walk(x, "expr",
func(f *File, x interface{}, context string) {
f.walk(x, ctxExpr,
func(f *File, x interface{}, context astContext) {
switch x.(type) {
case *ast.CallExpr:
found = true
@ -1110,7 +1054,17 @@ func (p *Package) rewriteRef(f *File) {
// Assign mangled names.
for _, n := range f.Name {
if n.Kind == "not-type" {
n.Kind = "var"
if n.Define == "" {
n.Kind = "var"
} else {
n.Kind = "macro"
n.FuncType = &FuncType{
Result: n.Type,
Go: &ast.FuncType{
Results: &ast.FieldList{List: []*ast.Field{{Type: n.Type.Go}}},
},
}
}
}
if n.Mangle == "" {
p.mangleName(n)
@ -1130,10 +1084,10 @@ func (p *Package) rewriteRef(f *File) {
}
var expr ast.Expr = ast.NewIdent(r.Name.Mangle) // default
switch r.Context {
case "call", "call2":
case ctxCall, ctxCall2:
if r.Name.Kind != "func" {
if r.Name.Kind == "type" {
r.Context = "type"
r.Context = ctxType
if r.Name.Type == nil {
error_(r.Pos(), "invalid conversion to C.%s: undefined C type '%s'", fixGo(r.Name.Go), r.Name.C)
break
@ -1145,7 +1099,7 @@ func (p *Package) rewriteRef(f *File) {
break
}
functions[r.Name.Go] = true
if r.Context == "call2" {
if r.Context == ctxCall2 {
if r.Name.Go == "_CMalloc" {
error_(r.Pos(), "no two-result form for C.malloc")
break
@ -1163,8 +1117,9 @@ func (p *Package) rewriteRef(f *File) {
r.Name = n
break
}
case "expr":
if r.Name.Kind == "func" {
case ctxExpr:
switch r.Name.Kind {
case "func":
if builtinDefs[r.Name.C] != "" {
error_(r.Pos(), "use of builtin '%s' not in function call", fixGo(r.Name.C))
}
@ -1191,25 +1146,25 @@ func (p *Package) rewriteRef(f *File) {
Fun: &ast.Ident{NamePos: (*r.Expr).Pos(), Name: "_Cgo_ptr"},
Args: []ast.Expr{ast.NewIdent(name.Mangle)},
}
} else if r.Name.Kind == "type" {
case "type":
// Okay - might be new(T)
if r.Name.Type == nil {
error_(r.Pos(), "expression C.%s: undefined C type '%s'", fixGo(r.Name.Go), r.Name.C)
break
}
expr = r.Name.Type.Go
} else if r.Name.Kind == "var" {
case "var":
expr = &ast.StarExpr{Star: (*r.Expr).Pos(), X: expr}
case "macro":
expr = &ast.CallExpr{Fun: expr}
}
case "selector":
case ctxSelector:
if r.Name.Kind == "var" {
expr = &ast.StarExpr{Star: (*r.Expr).Pos(), X: expr}
} else {
error_(r.Pos(), "only C variables allowed in selector expression %s", fixGo(r.Name.Go))
}
case "type":
case ctxType:
if r.Name.Kind != "type" {
error_(r.Pos(), "expression C.%s used as type", fixGo(r.Name.Go))
} else if r.Name.Type == nil {
@ -1224,6 +1179,7 @@ func (p *Package) rewriteRef(f *File) {
error_(r.Pos(), "must call C.%s", fixGo(r.Name.Go))
}
}
if *godefs {
// Substitute definition for mangled type name.
if id, ok := expr.(*ast.Ident); ok {
@ -1245,7 +1201,17 @@ func (p *Package) rewriteRef(f *File) {
expr = &ast.Ident{NamePos: pos, Name: x.Name}
}
// Change AST, because some later processing depends on it,
// and also because -godefs mode still prints the AST.
old := *r.Expr
*r.Expr = expr
// Record source-level edit for cgo output.
repl := gofmt(expr)
if r.Name.Kind != "type" {
repl = "(" + repl + ")"
}
f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), repl)
}
// Remove functions only used as expressions, so their respective
@ -1270,7 +1236,7 @@ func (p *Package) gccBaseCmd() []string {
if ret := strings.Fields(os.Getenv("GCC")); len(ret) > 0 {
return ret
}
return strings.Fields(defaultCC)
return strings.Fields(defaultCC(goos, goarch))
}
// gccMachine returns the gcc -m flag to use, either "-m32", "-m64" or "-marm".
@ -2186,6 +2152,12 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
name := c.Ident("_Ctype_" + dt.Name)
goIdent[name.Name] = name
sub := c.Type(dt.Type, pos)
if badPointerTypedef(dt.Name) {
// Treat this typedef as a uintptr.
s := *sub
s.Go = c.uintptr
sub = &s
}
t.Go = name
if unionWithPointer[sub.Go] {
unionWithPointer[t.Go] = true
@ -2266,7 +2238,7 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
if ss, ok := dwarfToName[s]; ok {
s = ss
}
s = strings.Join(strings.Split(s, " "), "") // strip spaces
s = strings.Replace(s, " ", "", -1)
name := c.Ident("_Ctype_" + s)
tt := *t
typedef[name.Name] = &tt
@ -2344,6 +2316,17 @@ func (c *typeConv) FuncArg(dtype dwarf.Type, pos token.Pos) *Type {
if _, void := base(ptr.Type).(*dwarf.VoidType); void {
break
}
// ...or the typedef is one in which we expect bad pointers.
// It will be a uintptr instead of *X.
if badPointerTypedef(dt.Name) {
break
}
// If we already know the typedef for t just use that.
// See issue 19832.
if def := typedef[t.Go.(*ast.Ident).Name]; def != nil {
break
}
t = c.Type(ptr, pos)
if t == nil {
@ -2500,7 +2483,9 @@ func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.Struct
size := t.Size
talign := t.Align
if f.BitSize > 0 {
if f.BitSize%8 != 0 {
switch f.BitSize {
case 8, 16, 32, 64:
default:
continue
}
size = f.BitSize / 8
@ -2676,3 +2661,51 @@ func fieldPrefix(fld []*ast.Field) string {
}
return prefix
}
// badPointerTypedef reports whether t is a C typedef that should not be considered a pointer in Go.
// A typedef is bad if C code sometimes stores non-pointers in this type.
// TODO: Currently our best solution is to find these manually and list them as
// they come up. A better solution is desired.
func badPointerTypedef(t string) bool {
// The real bad types are CFNumberRef and CFTypeRef.
// Sometimes non-pointers are stored in these types.
// CFTypeRef is a supertype of those, so it can have bad pointers in it as well.
// We return true for the other CF*Ref types just so casting between them is easier.
// See comment below for details about the bad pointers.
return goos == "darwin" && strings.HasPrefix(t, "CF") && strings.HasSuffix(t, "Ref")
}
// Comment from Darwin's CFInternal.h
/*
// Tagged pointer support
// Low-bit set means tagged object, next 3 bits (currently)
// define the tagged object class, next 4 bits are for type
// information for the specific tagged object class. Thus,
// the low byte is for type info, and the rest of a pointer
// (32 or 64-bit) is for payload, whatever the tagged class.
//
// Note that the specific integers used to identify the
// specific tagged classes can and will change from release
// to release (that's why this stuff is in CF*Internal*.h),
// as can the definition of type info vs payload above.
//
#if __LP64__
#define CF_IS_TAGGED_OBJ(PTR) ((uintptr_t)(PTR) & 0x1)
#define CF_TAGGED_OBJ_TYPE(PTR) ((uintptr_t)(PTR) & 0xF)
#else
#define CF_IS_TAGGED_OBJ(PTR) 0
#define CF_TAGGED_OBJ_TYPE(PTR) 0
#endif
enum {
kCFTaggedObjectID_Invalid = 0,
kCFTaggedObjectID_Atom = (0 << 1) + 1,
kCFTaggedObjectID_Undefined3 = (1 << 1) + 1,
kCFTaggedObjectID_Undefined2 = (2 << 1) + 1,
kCFTaggedObjectID_Integer = (3 << 1) + 1,
kCFTaggedObjectID_DateTS = (4 << 1) + 1,
kCFTaggedObjectID_ManagedObjectID = (5 << 1) + 1, // Core Data
kCFTaggedObjectID_Date = (6 << 1) + 1,
kCFTaggedObjectID_Undefined7 = (7 << 1) + 1,
};
*/

23
libgo/go/cmd/cgo/main.go
View file

@ -24,6 +24,9 @@ import (
"runtime"
"sort"
"strings"
"cmd/internal/edit"
"cmd/internal/objabi"
)
// A Package collects information about the package we're going to write.
@ -54,6 +57,12 @@ type File struct {
Calls []*Call // all calls to C.xxx in AST
ExpFunc []*ExpFunc // exported functions for this file
Name map[string]*Name // map from Go name to Name
NamePos map[*Name]token.Pos // map from Name to position of the first reference
Edit *edit.Buffer
}
func (f *File) offset(p token.Pos) int {
return fset.Position(p).Offset
}
func nameKeys(m map[string]*Name) []string {
@ -75,7 +84,7 @@ type Call struct {
type Ref struct {
Name *Name
Expr *ast.Expr
Context string // "type", "expr", "call", or "call2"
Context astContext
}
func (r *Ref) Pos() token.Pos {
@ -88,7 +97,7 @@ type Name struct {
Mangle string // name used in generated Go
C string // name used in C
Define string // #define expansion
Kind string // "iconst", "uconst", "fconst", "sconst", "type", "var", "fpvar", "func", "not-type"
Kind string // "iconst", "fconst", "sconst", "type", "var", "fpvar", "func", "macro", "not-type"
Type *Type // the type of xxx
FuncType *FuncType
AddError bool
@ -100,12 +109,12 @@ func (n *Name) IsVar() bool {
return n.Kind == "var" || n.Kind == "fpvar"
}
// IsConst reports whether Kind is either "iconst", "uconst", "fconst" or "sconst"
// IsConst reports whether Kind is either "iconst", "fconst" or "sconst"
func (n *Name) IsConst() bool {
return strings.HasSuffix(n.Kind, "const")
}
// A ExpFunc is an exported function, callable from C.
// An ExpFunc is an exported function, callable from C.
// Such functions are identified in the Go input file
// by doc comments containing the line //export ExpName
type ExpFunc struct {
@ -214,6 +223,7 @@ var importSyscall = flag.Bool("import_syscall", true, "import syscall in generat
var goarch, goos string
func main() {
objabi.AddVersionFlag() // -V
flag.Usage = usage
flag.Parse()
@ -294,6 +304,7 @@ func main() {
}
f := new(File)
f.Edit = edit.NewBuffer(b)
f.ParseGo(input, b)
f.DiscardCgoDirectives()
fs[i] = f
@ -314,11 +325,13 @@ func main() {
p.Translate(f)
for _, cref := range f.Ref {
switch cref.Context {
case "call", "call2":
case ctxCall, ctxCall2:
if cref.Name.Kind != "type" {
break
}
old := *cref.Expr
*cref.Expr = cref.Name.Type.Go
f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
}
}
if nerrors > 0 {

104
libgo/go/cmd/cgo/out.go
View file

@ -16,6 +16,7 @@ import (
"go/token"
"io"
"os"
"path/filepath"
"sort"
"strings"
)
@ -116,7 +117,13 @@ func (p *Package) writeDefs() {
// Which is not useful. Moreover we never override source info,
// so subsequent source code uses the same source info.
// Moreover, empty file name makes compile emit no source debug info at all.
noSourceConf.Fprint(fgo2, fset, def.Go)
var buf bytes.Buffer
noSourceConf.Fprint(&buf, fset, def.Go)
if bytes.HasPrefix(buf.Bytes(), []byte("_Ctype_")) {
// This typedef is of the form `typedef a b` and should be an alias.
fmt.Fprintf(fgo2, "= ")
}
fmt.Fprintf(fgo2, "%s", buf.Bytes())
fmt.Fprintf(fgo2, "\n\n")
}
if *gccgo {
@ -424,10 +431,12 @@ func (p *Package) writeDefsFunc(fgo2 io.Writer, n *Name, callsMalloc *bool) {
inProlog := builtinDefs[name] != ""
cname := fmt.Sprintf("_cgo%s%s", cPrefix, n.Mangle)
paramnames := []string(nil)
for i, param := range d.Type.Params.List {
paramName := fmt.Sprintf("p%d", i)
param.Names = []*ast.Ident{ast.NewIdent(paramName)}
paramnames = append(paramnames, paramName)
if d.Type.Params != nil {
for i, param := range d.Type.Params.List {
paramName := fmt.Sprintf("p%d", i)
param.Names = []*ast.Ident{ast.NewIdent(paramName)}
paramnames = append(paramnames, paramName)
}
}
if *gccgo {
@ -526,8 +535,10 @@ func (p *Package) writeDefsFunc(fgo2 io.Writer, n *Name, callsMalloc *bool) {
fmt.Fprintf(fgo2, "\tif errno != 0 { r2 = syscall.Errno(errno) }\n")
}
fmt.Fprintf(fgo2, "\tif _Cgo_always_false {\n")
for i := range d.Type.Params.List {
fmt.Fprintf(fgo2, "\t\t_Cgo_use(p%d)\n", i)
if d.Type.Params != nil {
for i := range d.Type.Params.List {
fmt.Fprintf(fgo2, "\t\t_Cgo_use(p%d)\n", i)
}
}
fmt.Fprintf(fgo2, "\t}\n")
fmt.Fprintf(fgo2, "\treturn\n")
@ -540,7 +551,7 @@ func (p *Package) writeOutput(f *File, srcfile string) {
if strings.HasSuffix(base, ".go") {
base = base[0 : len(base)-3]
}
base = strings.Map(slashToUnderscore, base)
base = filepath.Base(base)
fgo1 := creat(*objDir + base + ".cgo1.go")
fgcc := creat(*objDir + base + ".cgo2.c")
@ -549,10 +560,12 @@ func (p *Package) writeOutput(f *File, srcfile string) {
// Write Go output: Go input with rewrites of C.xxx to _C_xxx.
fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n\n")
conf.Fprint(fgo1, fset, f.AST)
fmt.Fprintf(fgo1, "//line %s:1\n", srcfile)
fgo1.Write(f.Edit.Bytes())
// While we process the vars and funcs, also write gcc output.
// Gcc output starts with the preamble.
fmt.Fprintf(fgcc, "%s\n", builtinProlog)
fmt.Fprintf(fgcc, "%s\n", f.Preamble)
fmt.Fprintf(fgcc, "%s\n", gccProlog)
fmt.Fprintf(fgcc, "%s\n", tsanProlog)
@ -639,14 +652,18 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
fmt.Fprint(fgcc, "(__typeof__(a->r)) ")
}
}
fmt.Fprintf(fgcc, "%s(", n.C)
for i := range n.FuncType.Params {
if i > 0 {
fmt.Fprintf(fgcc, ", ")
if n.Kind == "macro" {
fmt.Fprintf(fgcc, "%s;\n", n.C)
} else {
fmt.Fprintf(fgcc, "%s(", n.C)
for i := range n.FuncType.Params {
if i > 0 {
fmt.Fprintf(fgcc, ", ")
}
fmt.Fprintf(fgcc, "a->p%d", i)
}
fmt.Fprintf(fgcc, "a->p%d", i)
fmt.Fprintf(fgcc, ");\n")
}
fmt.Fprintf(fgcc, ");\n")
if n.AddError {
fmt.Fprintf(fgcc, "\t_cgo_errno = errno;\n")
}
@ -702,14 +719,18 @@ func (p *Package) writeGccgoOutputFunc(fgcc *os.File, n *Name) {
fmt.Fprintf(fgcc, "(void*)")
}
}
fmt.Fprintf(fgcc, "%s(", n.C)
for i := range n.FuncType.Params {
if i > 0 {
fmt.Fprintf(fgcc, ", ")
if n.Kind == "macro" {
fmt.Fprintf(fgcc, "%s;\n", n.C)
} else {
fmt.Fprintf(fgcc, "%s(", n.C)
for i := range n.FuncType.Params {
if i > 0 {
fmt.Fprintf(fgcc, ", ")
}
fmt.Fprintf(fgcc, "p%d", i)
}
fmt.Fprintf(fgcc, "p%d", i)
fmt.Fprintf(fgcc, ");\n")
}
fmt.Fprintf(fgcc, ");\n")
fmt.Fprintf(fgcc, "\t_cgo_tsan_release();\n")
if t := n.FuncType.Result; t != nil {
fmt.Fprintf(fgcc, "\treturn ")
@ -1009,7 +1030,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
default:
// Declare a result struct.
fmt.Fprintf(fgcch, "\n/* Return type for %s */\n", exp.ExpName)
fmt.Fprintf(fgcch, "struct %s_result {\n", exp.ExpName)
fmt.Fprintf(fgcch, "struct %s_return {\n", exp.ExpName)
forFieldList(fntype.Results,
func(i int, aname string, atype ast.Expr) {
t := p.cgoType(atype)
@ -1020,7 +1041,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
fmt.Fprint(fgcch, "\n")
})
fmt.Fprintf(fgcch, "};\n")
fmt.Fprintf(cdeclBuf, "struct %s_result", exp.ExpName)
fmt.Fprintf(cdeclBuf, "struct %s_return", exp.ExpName)
}
cRet := cdeclBuf.String()
@ -1046,7 +1067,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
fmt.Fprintf(fgcch, "\n%s", exp.Doc)
}
fmt.Fprintf(fgcch, "extern %s %s %s;\n", cRet, exp.ExpName, cParams)
fmt.Fprintf(fgcch, "extern %s %s%s;\n", cRet, exp.ExpName, cParams)
// We need to use a name that will be exported by the
// Go code; otherwise gccgo will make it static and we
@ -1155,6 +1176,7 @@ func (p *Package) writeExportHeader(fgcch io.Writer) {
pkg = p.PackagePath
}
fmt.Fprintf(fgcch, "/* package %s */\n\n", pkg)
fmt.Fprintf(fgcch, "%s\n", builtinExportProlog)
fmt.Fprintf(fgcch, "/* Start of preamble from import \"C\" comments. */\n\n")
fmt.Fprintf(fgcch, "%s\n", p.Preamble)
@ -1247,8 +1269,9 @@ func (p *Package) cgoType(e ast.Expr) *Type {
// Slice: pointer, len, cap.
return &Type{Size: p.PtrSize * 3, Align: p.PtrSize, C: c("GoSlice")}
}
// Non-slice array types are not supported.
case *ast.StructType:
// TODO
// Not supported.
case *ast.FuncType:
return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("void*")}
case *ast.InterfaceType:
@ -1398,7 +1421,7 @@ const builtinProlog = `
/* Define intgo when compiling with GCC. */
typedef ptrdiff_t intgo;
typedef struct { char *p; intgo n; } _GoString_;
typedef struct { const char *p; intgo n; } _GoString_;
typedef struct { char *p; intgo n; intgo c; } _GoBytes_;
_GoString_ GoString(char *p);
_GoString_ GoStringN(char *p, int l);
@ -1406,6 +1429,12 @@ _GoBytes_ GoBytes(void *p, int n);
char *CString(_GoString_);
void *CBytes(_GoBytes_);
void *_CMalloc(size_t);
__attribute__ ((unused))
static size_t _GoStringLen(_GoString_ s) { return s.n; }
__attribute__ ((unused))
static const char *_GoStringPtr(_GoString_ s) { return s.p; }
`
const goProlog = `
@ -1637,6 +1666,27 @@ void localCgoCheckResult(Eface val) {
}
`
// builtinExportProlog is a shorter version of builtinProlog,
// to be put into the _cgo_export.h file.
// For historical reasons we can't use builtinProlog in _cgo_export.h,
// because _cgo_export.h defines GoString as a struct while builtinProlog
// defines it as a function. We don't change this to avoid unnecessarily
// breaking existing code.
const builtinExportProlog = `
#line 1 "cgo-builtin-prolog"
#include <stddef.h> /* for ptrdiff_t below */
#ifndef GO_CGO_EXPORT_PROLOGUE_H
#define GO_CGO_EXPORT_PROLOGUE_H
typedef ptrdiff_t intgo;
typedef struct { const char *p; intgo n; } _GoString_;
#endif
`
func (p *Package) gccExportHeaderProlog() string {
return strings.Replace(gccExportHeaderProlog, "GOINTBITS", fmt.Sprint(8*p.IntSize), -1)
}
@ -1670,7 +1720,7 @@ typedef double _Complex GoComplex128;
*/
typedef char _check_for_GOINTBITS_bit_pointer_matching_GoInt[sizeof(void*)==GOINTBITS/8 ? 1:-1];
typedef struct { const char *p; GoInt n; } GoString;
typedef _GoString_ GoString;
typedef void *GoMap;
typedef void *GoChan;
typedef struct { void *t; void *v; } GoInterface;

145
libgo/go/cmd/go/alldocs.go
View file

@ -14,12 +14,12 @@
// The commands are:
//
// build compile packages and dependencies
// clean remove object files
// clean remove object files and cached files
// doc show documentation for package or symbol
// env print Go environment information
// bug start a bug report
// fix run go tool fix on packages
// fmt run gofmt on package sources
// fix update packages to use new APIs
// fmt gofmt (reformat) package sources
// generate generate Go files by processing source
// get download and install packages and dependencies
// install compile and install packages and dependencies
@ -28,7 +28,7 @@
// test test packages
// tool run specified go tool
// version print Go version
// vet run go tool vet on packages
// vet report likely mistakes in packages
//
// Use "go help [command]" for more information about a command.
//
@ -104,15 +104,15 @@
// -x
// print the commands.
//
// -asmflags 'flag list'
// -asmflags '[pattern=]arg list'
// arguments to pass on each go tool asm invocation.
// -buildmode mode
// build mode to use. See 'go help buildmode' for more.
// -compiler name
// name of compiler to use, as in runtime.Compiler (gccgo or gc).
// -gccgoflags 'arg list'
// -gccgoflags '[pattern=]arg list'
// arguments to pass on each gccgo compiler/linker invocation.
// -gcflags 'arg list'
// -gcflags '[pattern=]arg list'
// arguments to pass on each go tool compile invocation.
// -installsuffix suffix
// a suffix to use in the name of the package installation directory,
@ -121,7 +121,7 @@
// or, if set explicitly, has _race appended to it. Likewise for the -msan
// flag. Using a -buildmode option that requires non-default compile flags
// has a similar effect.
// -ldflags 'flag list'
// -ldflags '[pattern=]arg list'
// arguments to pass on each go tool link invocation.
// -linkshared
// link against shared libraries previously created with
@ -139,9 +139,21 @@
// For example, instead of running asm, the go command will run
// 'cmd args /path/to/asm <arguments for asm>'.
//
// All the flags that take a list of arguments accept a space-separated
// list of strings. To embed spaces in an element in the list, surround
// it with either single or double quotes.
// The -asmflags, -gccgoflags, -gcflags, and -ldflags flags accept a
// space-separated list of arguments to pass to an underlying tool
// during the build. To embed spaces in an element in the list, surround
// it with either single or double quotes. The argument list may be
// preceded by a package pattern and an equal sign, which restricts
// the use of that argument list to the building of packages matching
// that pattern (see 'go help packages' for a description of package
// patterns). Without a pattern, the argument list applies only to the
// packages named on the command line. The flags may be repeated
// with different patterns in order to specify different arguments for
// different sets of packages. If a package matches patterns given in
// multiple flags, the latest match on the command line wins.
// For example, 'go build -gcflags=-S fmt' prints the disassembly
// only for package fmt, while 'go build -gcflags=all=-S fmt'
// prints the disassembly for fmt and all its dependencies.
//
// For more about specifying packages, see 'go help packages'.
// For more about where packages and binaries are installed,
@ -158,11 +170,11 @@
// See also: go install, go get, go clean.
//
//
// Remove object files
// Remove object files and cached files
//
// Usage:
//
// go clean [-i] [-r] [-n] [-x] [build flags] [packages]
// go clean [-i] [-r] [-n] [-x] [-cache] [-testcache] [build flags] [packages]
//
// Clean removes object files from package source directories.
// The go command builds most objects in a temporary directory,
@ -200,6 +212,11 @@
//
// The -x flag causes clean to print remove commands as it executes them.
//
// The -cache flag causes clean to remove the entire go build cache.
//
// The -testcache flag causes clean to expire all test results in the
// go build cache.
//
// For more about build flags, see 'go help build'.
//
// For more about specifying packages, see 'go help packages'.
@ -328,6 +345,8 @@
// The -json flag prints the environment in JSON format
// instead of as a shell script.
//
// For more about environment variables, see 'go help environment'.
//
//
// Start a bug report
//
@ -339,7 +358,7 @@
// The report includes useful system information.
//
//
// Run go tool fix on packages
// Update packages to use new APIs
//
// Usage:
//
@ -355,7 +374,7 @@
// See also: go fmt, go vet.
//
//
// Run gofmt on package sources
// Gofmt (reformat) package sources
//
// Usage:
//
@ -543,10 +562,11 @@
//
// Usage:
//
// go install [build flags] [packages]
// go install [-i] [build flags] [packages]
//
// Install compiles and installs the packages named by the import paths,
// along with their dependencies.
// Install compiles and installs the packages named by the import paths.
//
// The -i flag installs the dependencies of the named packages as well.
//
// For more about the build flags, see 'go help build'.
// For more about specifying packages, see 'go help packages'.
@ -719,10 +739,10 @@
//
// 'Go test' recompiles each package along with any files with names matching
// the file pattern "*_test.go".
// Files whose names begin with "_" (including "_test.go") or "." are ignored.
// These additional files can contain test functions, benchmark functions, and
// example functions. See 'go help testfunc' for more.
// Each listed package causes the execution of a separate test binary.
// Files whose names begin with "_" (including "_test.go") or "." are ignored.
//
// Test files that declare a package with the suffix "_test" will be compiled as a
// separate package, and then linked and run with the main test binary.
@ -730,11 +750,46 @@
// The go tool will ignore a directory named "testdata", making it available
// to hold ancillary data needed by the tests.
//
// By default, go test needs no arguments. It compiles and tests the package
// with source in the current directory, including tests, and runs the tests.
// As part of building a test binary, go test runs go vet on the package
// and its test source files to identify significant problems. If go vet
// finds any problems, go test reports those and does not run the test binary.
// Only a high-confidence subset of the default go vet checks are used.
// To disable the running of go vet, use the -vet=off flag.
//
// The package is built in a temporary directory so it does not interfere with the
// non-test installation.
// Go test runs in two different modes: local directory mode when invoked with
// no package arguments (for example, 'go test'), and package list mode when
// invoked with package arguments (for example 'go test math', 'go test ./...',
// and even 'go test .').
//
// In local directory mode, go test compiles and tests the package sources
// found in the current directory and then runs the resulting test binary.
// In this mode, caching (discussed below) is disabled. After the package test
// finishes, go test prints a summary line showing the test status ('ok' or 'FAIL'),
// package name, and elapsed time.
//
// In package list mode, go test compiles and tests each of the packages
// listed on the command line. If a package test passes, go test prints only
// the final 'ok' summary line. If a package test fails, go test prints the
// full test output. If invoked with the -bench or -v flag, go test prints
// the full output even for passing package tests, in order to display the
// requested benchmark results or verbose logging.
//
// All test output and summary lines are printed to the go command's standard
// output, even if the test printed them to its own standard error.
// (The go command's standard error is reserved for printing errors building
// the tests.)
//
// In package list mode, go test also caches successful package test results.
// If go test has cached a previous test run using the same test binary and
// the same command line consisting entirely of cacheable test flags
// (defined as -cpu, -list, -parallel, -run, -short, and -v),
// go test will redisplay the previous output instead of running the test
// binary again. In the summary line, go test prints '(cached)' in place of
// the elapsed time. To disable test caching, use any test flag or argument
// other than the cacheable flags. The idiomatic way to disable test caching
// explicitly is to use -count=1. A cached result is treated as executing in
// no time at all, so a successful package test result will be cached and reused
// regardless of -timeout setting.
//
// In addition to the build flags, the flags handled by 'go test' itself are:
//
@ -757,6 +812,10 @@
// Install packages that are dependencies of the test.
// Do not run the test.
//
// -json
// Convert test output to JSON suitable for automated processing.
// See 'go doc test2json' for the encoding details.
//
// -o file
// Compile the test binary to the named file.
// The test still runs (unless -c or -i is specified).
@ -782,7 +841,7 @@
// The -n flag causes tool to print the command that would be
// executed but not execute it.
//
// For more about each tool command, see 'go tool command -h'.
// For more about each tool command, see 'go doc cmd/<command>'.
//
//
// Print Go version
@ -794,7 +853,7 @@
// Version prints the Go version, as reported by runtime.Version.
//
//
// Run go tool vet on packages
// Report likely mistakes in packages
//
// Usage:
//
@ -808,7 +867,9 @@
// The -n flag prints commands that would be executed.
// The -x flag prints commands as they are executed.
//
// For more about build flags, see 'go help build'.
// The build flags supported by go vet are those that control package resolution
// and execution, such as -n, -x, -v, -tags, and -toolexec.
// For more about these flags, see 'go help build'.
//
// See also: go fmt, go fix.
//
@ -917,8 +978,10 @@
// comment, indicating that the package sources are included
// for documentation only and must not be used to build the
// package binary. This enables distribution of Go packages in
// their compiled form alone. See the go/build package documentation
// for more details.
// their compiled form alone. Even binary-only packages require
// accurate import blocks listing required dependencies, so that
// those dependencies can be supplied when linking the resulting
// command.
//
//
// GOPATH environment variable
@ -1096,6 +1159,12 @@
// See https://golang.org/doc/articles/race_detector.html.
// GOROOT
// The root of the go tree.
// GOTMPDIR
// The directory where the go command will write
// temporary source files, packages, and binaries.
// GOCACHE
// The directory where the go command will store
// cached information for reuse in future builds.
//
// Environment variables for use with cgo:
//
@ -1130,6 +1199,9 @@
// GO386
// For GOARCH=386, the floating point instruction set.
// Valid values are 387, sse2.
// GOMIPS
// For GOARCH=mips{,le}, whether to use floating point instructions.
// Valid values are hardfloat (default), softfloat.
//
// Special-purpose environment variables:
//
@ -1460,10 +1532,10 @@
// significantly more expensive.
// Sets -cover.
//
// -coverpkg pkg1,pkg2,pkg3
// Apply coverage analysis in each test to the given list of packages.
// -coverpkg pattern1,pattern2,pattern3
// Apply coverage analysis in each test to packages matching the patterns.
// The default is for each test to analyze only the package being tested.
// Packages are specified as import paths.
// See 'go help packages' for a description of package patterns.
// Sets -cover.
//
// -cpu 1,2,4
@ -1471,6 +1543,9 @@
// benchmarks should be executed. The default is the current value
// of GOMAXPROCS.
//
// -failfast
// Do not start new tests after the first test failure.
//
// -list regexp
// List tests, benchmarks, or examples matching the regular expression.
// No tests, benchmarks or examples will be run. This will only
@ -1503,12 +1578,20 @@
//
// -timeout d
// If a test binary runs longer than duration d, panic.
// If d is 0, the timeout is disabled.
// The default is 10 minutes (10m).
//
// -v
// Verbose output: log all tests as they are run. Also print all
// text from Log and Logf calls even if the test succeeds.
//
// -vet list
// Configure the invocation of "go vet" during "go test"
// to use the comma-separated list of vet checks.
// If list is empty, "go test" runs "go vet" with a curated list of
// checks believed to be always worth addressing.
// If list is "off", "go test" does not run "go vet" at all.
//
// The following flags are also recognized by 'go test' and can be used to
// profile the tests during execution:
//

1315
libgo/go/cmd/go/go_test.go
View file

File diff suppressed because it is too large Load diff

81
libgo/go/cmd/go/go_windows_test.go
View file

@ -5,12 +5,14 @@
package main
import (
"fmt"
"internal/testenv"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"testing"
)
@ -54,3 +56,82 @@ func TestAbsolutePath(t *testing.T) {
t.Fatalf("wrong output found: %v %v", err, string(output))
}
}
func isWindowsXP(t *testing.T) bool {
v, err := syscall.GetVersion()
if err != nil {
t.Fatalf("GetVersion failed: %v", err)
}
major := byte(v)
return major < 6
}
func runIcacls(t *testing.T, args ...string) string {
t.Helper()
out, err := exec.Command("icacls", args...).CombinedOutput()
if err != nil {
t.Fatalf("icacls failed: %v\n%v", err, string(out))
}
return string(out)
}
func runGetACL(t *testing.T, path string) string {
t.Helper()
cmd := fmt.Sprintf(`Get-Acl "%s" | Select -expand AccessToString`, path)
out, err := exec.Command("powershell", "-Command", cmd).CombinedOutput()
if err != nil {
t.Fatalf("Get-Acl failed: %v\n%v", err, string(out))
}
return string(out)
}
// For issue 22343: verify that executable file created by "go build" command
// has discretionary access control list (DACL) set as if the file
// was created in the destination directory.
func TestACL(t *testing.T) {
if isWindowsXP(t) {
t.Skip("Windows XP does not have powershell command")
}
tmpdir, err := ioutil.TempDir("", "TestACL")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
newtmpdir := filepath.Join(tmpdir, "tmp")
err = os.Mkdir(newtmpdir, 0777)
if err != nil {
t.Fatal(err)
}
// When TestACL/tmp directory is created, it will have
// the same security attributes as TestACL.
// Add Guest account full access to TestACL/tmp - this
// will make all files created in TestACL/tmp have different
// security attributes to the files created in TestACL.
runIcacls(t, newtmpdir,
"/grant", "guest:(oi)(ci)f", // add Guest user to have full access
)
src := filepath.Join(tmpdir, "main.go")
err = ioutil.WriteFile(src, []byte("package main; func main() { }\n"), 0644)
if err != nil {
t.Fatal(err)
}
exe := filepath.Join(tmpdir, "main.exe")
cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", exe, src)
cmd.Env = append(os.Environ(),
"TMP="+newtmpdir,
"TEMP="+newtmpdir,
)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("go command failed: %v\n%v", err, string(out))
}
// exe file is expected to have the same security attributes as the src.
if got, expected := runGetACL(t, exe), runGetACL(t, src); got != expected {
t.Fatalf("expected Get-Acl output of \n%v\n, got \n%v\n", expected, got)
}
}

4
libgo/go/cmd/go/internal/base/base.go
View file

@ -62,8 +62,8 @@ func (c *Command) Name() string {
}
func (c *Command) Usage() {
fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.UsageLine)
fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(c.Long))
fmt.Fprintf(os.Stderr, "usage: %s\n", c.UsageLine)
fmt.Fprintf(os.Stderr, "Run 'go help %s' for details.\n", c.Name())
os.Exit(2)
}

22
libgo/go/cmd/go/internal/base/path.go
View file

@ -44,28 +44,6 @@ func RelPaths(paths []string) []string {
return out
}
// FilterDotUnderscoreFiles returns a slice containing all elements
// of path whose base name doesn't begin with "." or "_".
func FilterDotUnderscoreFiles(path []string) []string {
var out []string // lazily initialized
for i, p := range path {
base := filepath.Base(p)
if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") {
if out == nil {
out = append(make([]string, 0, len(path)), path[:i]...)
}
continue
}
if out != nil {
out = append(out, p)
}
}
if out == nil {
return path
}
return out
}
// IsTestFile reports whether the source file is a set of tests and should therefore
// be excluded from coverage analysis.
func IsTestFile(file string) bool {

11
libgo/go/cmd/go/internal/base/tool.go
View file

@ -36,18 +36,9 @@ func Tool(toolName string) string {
}
// Give a nice message if there is no tool with that name.
if _, err := os.Stat(toolPath); err != nil {
if isInGoToolsRepo(toolName) {
fmt.Fprintf(os.Stderr, "go tool: no such tool %q; to install:\n\tgo get golang.org/x/tools/cmd/%s\n", toolName, toolName)
} else {
fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", toolName)
}
fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", toolName)
SetExitStatus(2)
Exit()
}
return toolPath
}
// TODO: Delete.
func isInGoToolsRepo(toolName string) bool {
return false
}

453
libgo/go/cmd/go/internal/cache/cache.go vendored Normal file
View file

@ -0,0 +1,453 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package cache implements a build artifact cache.
package cache
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
// An ActionID is a cache action key, the hash of a complete description of a
// repeatable computation (command line, environment variables,
// input file contents, executable contents).
type ActionID [HashSize]byte
// An OutputID is a cache output key, the hash of an output of a computation.
type OutputID [HashSize]byte
// A Cache is a package cache, backed by a file system directory tree.
type Cache struct {
dir string
log *os.File
now func() time.Time
}
// Open opens and returns the cache in the given directory.
//
// It is safe for multiple processes on a single machine to use the
// same cache directory in a local file system simultaneously.
// They will coordinate using operating system file locks and may
// duplicate effort but will not corrupt the cache.
//
// However, it is NOT safe for multiple processes on different machines
// to share a cache directory (for example, if the directory were stored
// in a network file system). File locking is notoriously unreliable in
// network file systems and may not suffice to protect the cache.
//
func Open(dir string) (*Cache, error) {
info, err := os.Stat(dir)
if err != nil {
return nil, err
}
if !info.IsDir() {
return nil, &os.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")}
}
for i := 0; i < 256; i++ {
name := filepath.Join(dir, fmt.Sprintf("%02x", i))
if err := os.MkdirAll(name, 0777); err != nil {
return nil, err
}
}
f, err := os.OpenFile(filepath.Join(dir, "log.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
return nil, err
}
c := &Cache{
dir: dir,
log: f,
now: time.Now,
}
return c, nil
}
// fileName returns the name of the file corresponding to the given id.
func (c *Cache) fileName(id [HashSize]byte, key string) string {
return filepath.Join(c.dir, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+"-"+key)
}
var errMissing = errors.New("cache entry not found")
const (
// action entry file is "v1 <hex id> <hex out> <decimal size space-padded to 20 bytes> <unixnano space-padded to 20 bytes>\n"
hexSize = HashSize * 2
entrySize = 2 + 1 + hexSize + 1 + hexSize + 1 + 20 + 1 + 20 + 1
)
// verify controls whether to run the cache in verify mode.
// In verify mode, the cache always returns errMissing from Get
// but then double-checks in Put that the data being written
// exactly matches any existing entry. This provides an easy
// way to detect program behavior that would have been different
// had the cache entry been returned from Get.
//
// verify is enabled by setting the environment variable
// GODEBUG=gocacheverify=1.
var verify = false
func init() { initEnv() }
func initEnv() {
verify = false
debugHash = false
debug := strings.Split(os.Getenv("GODEBUG"), ",")
for _, f := range debug {
if f == "gocacheverify=1" {
verify = true
}
if f == "gocachehash=1" {
debugHash = true
}
}
}
// Get looks up the action ID in the cache,
// returning the corresponding output ID and file size, if any.
// Note that finding an output ID does not guarantee that the
// saved file for that output ID is still available.
func (c *Cache) Get(id ActionID) (Entry, error) {
if verify {
return Entry{}, errMissing
}
return c.get(id)
}
type Entry struct {
OutputID OutputID
Size int64
Time time.Time
}
// get is Get but does not respect verify mode, so that Put can use it.
func (c *Cache) get(id ActionID) (Entry, error) {
missing := func() (Entry, error) {
fmt.Fprintf(c.log, "%d miss %x\n", c.now().Unix(), id)
return Entry{}, errMissing
}
f, err := os.Open(c.fileName(id, "a"))
if err != nil {
return missing()
}
defer f.Close()
entry := make([]byte, entrySize+1) // +1 to detect whether f is too long
if n, err := io.ReadFull(f, entry); n != entrySize || err != io.ErrUnexpectedEOF {
return missing()
}
if entry[0] != 'v' || entry[1] != '1' || entry[2] != ' ' || entry[3+hexSize] != ' ' || entry[3+hexSize+1+hexSize] != ' ' || entry[3+hexSize+1+hexSize+1+20] != ' ' || entry[entrySize-1] != '\n' {
return missing()
}
eid, entry := entry[3:3+hexSize], entry[3+hexSize:]
eout, entry := entry[1:1+hexSize], entry[1+hexSize:]
esize, entry := entry[1:1+20], entry[1+20:]
etime, entry := entry[1:1+20], entry[1+20:]
var buf [HashSize]byte
if _, err := hex.Decode(buf[:], eid); err != nil || buf != id {
return missing()
}
if _, err := hex.Decode(buf[:], eout); err != nil {
return missing()
}
i := 0
for i < len(esize) && esize[i] == ' ' {
i++
}
size, err := strconv.ParseInt(string(esize[i:]), 10, 64)
if err != nil || size < 0 {
return missing()
}
i = 0
for i < len(etime) && etime[i] == ' ' {
i++
}
tm, err := strconv.ParseInt(string(etime[i:]), 10, 64)
if err != nil || size < 0 {
return missing()
}
fmt.Fprintf(c.log, "%d get %x\n", c.now().Unix(), id)
c.used(c.fileName(id, "a"))
return Entry{buf, size, time.Unix(0, tm)}, nil
}
// GetBytes looks up the action ID in the cache and returns
// the corresponding output bytes.
// GetBytes should only be used for data that can be expected to fit in memory.
func (c *Cache) GetBytes(id ActionID) ([]byte, Entry, error) {
entry, err := c.Get(id)
if err != nil {
return nil, entry, err
}
data, _ := ioutil.ReadFile(c.OutputFile(entry.OutputID))
if sha256.Sum256(data) != entry.OutputID {
return nil, entry, errMissing
}
return data, entry, nil
}
// OutputFile returns the name of the cache file storing output with the given OutputID.
func (c *Cache) OutputFile(out OutputID) string {
file := c.fileName(out, "d")
c.used(file)
return file
}
// Time constants for cache expiration.
//
// We set the mtime on a cache file on each use, but at most one per mtimeInterval (1 hour),
// to avoid causing many unnecessary inode updates. The mtimes therefore
// roughly reflect "time of last use" but may in fact be older by at most an hour.
//
// We scan the cache for entries to delete at most once per trimInterval (1 day).
//
// When we do scan the cache, we delete entries that have not been used for
// at least trimLimit (5 days). Statistics gathered from a month of usage by
// Go developers found that essentially all reuse of cached entries happened
// within 5 days of the previous reuse. See golang.org/issue/22990.
const (
mtimeInterval = 1 * time.Hour
trimInterval = 24 * time.Hour
trimLimit = 5 * 24 * time.Hour
)
// used makes a best-effort attempt to update mtime on file,
// so that mtime reflects cache access time.
//
// Because the reflection only needs to be approximate,
// and to reduce the amount of disk activity caused by using
// cache entries, used only updates the mtime if the current
// mtime is more than an hour old. This heuristic eliminates
// nearly all of the mtime updates that would otherwise happen,
// while still keeping the mtimes useful for cache trimming.
func (c *Cache) used(file string) {
info, err := os.Stat(file)
if err == nil && c.now().Sub(info.ModTime()) < mtimeInterval {
return
}
os.Chtimes(file, c.now(), c.now())
}
// Trim removes old cache entries that are likely not to be reused.
func (c *Cache) Trim() {
now := c.now()
// We maintain in dir/trim.txt the time of the last completed cache trim.
// If the cache has been trimmed recently enough, do nothing.
// This is the common case.
data, _ := ioutil.ReadFile(filepath.Join(c.dir, "trim.txt"))
t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
if err == nil && now.Sub(time.Unix(t, 0)) < trimInterval {
return
}
// Trim each of the 256 subdirectories.
// We subtract an additional mtimeInterval
// to account for the imprecision of our "last used" mtimes.
cutoff := now.Add(-trimLimit - mtimeInterval)
for i := 0; i < 256; i++ {
subdir := filepath.Join(c.dir, fmt.Sprintf("%02x", i))
c.trimSubdir(subdir, cutoff)
}
ioutil.WriteFile(filepath.Join(c.dir, "trim.txt"), []byte(fmt.Sprintf("%d", now.Unix())), 0666)
}
// trimSubdir trims a single cache subdirectory.
func (c *Cache) trimSubdir(subdir string, cutoff time.Time) {
// Read all directory entries from subdir before removing
// any files, in case removing files invalidates the file offset
// in the directory scan. Also, ignore error from f.Readdirnames,
// because we don't care about reporting the error and we still
// want to process any entries found before the error.
f, err := os.Open(subdir)
if err != nil {
return
}
names, _ := f.Readdirnames(-1)
f.Close()
for _, name := range names {
// Remove only cache entries (xxxx-a and xxxx-d).
if !strings.HasSuffix(name, "-a") && !strings.HasSuffix(name, "-d") {
continue
}
entry := filepath.Join(subdir, name)
info, err := os.Stat(entry)
if err == nil && info.ModTime().Before(cutoff) {
os.Remove(entry)
}
}
}
// putIndexEntry adds an entry to the cache recording that executing the action
// with the given id produces an output with the given output id (hash) and size.
func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify bool) error {
// Note: We expect that for one reason or another it may happen
// that repeating an action produces a different output hash
// (for example, if the output contains a time stamp or temp dir name).
// While not ideal, this is also not a correctness problem, so we
// don't make a big deal about it. In particular, we leave the action
// cache entries writable specifically so that they can be overwritten.
//
// Setting GODEBUG=gocacheverify=1 does make a big deal:
// in verify mode we are double-checking that the cache entries
// are entirely reproducible. As just noted, this may be unrealistic
// in some cases but the check is also useful for shaking out real bugs.
entry := []byte(fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano()))
if verify && allowVerify {
old, err := c.get(id)
if err == nil && (old.OutputID != out || old.Size != size) {
// panic to show stack trace, so we can see what code is generating this cache entry.
msg := fmt.Sprintf("go: internal cache error: cache verify failed: id=%x changed:<<<\n%s\n>>>\nold: %x %d\nnew: %x %d", id, reverseHash(id), out, size, old.OutputID, old.Size)
panic(msg)
}
}
file := c.fileName(id, "a")
if err := ioutil.WriteFile(file, entry, 0666); err != nil {
os.Remove(file)
return err
}
os.Chtimes(file, c.now(), c.now()) // mainly for tests
fmt.Fprintf(c.log, "%d put %x %x %d\n", c.now().Unix(), id, out, size)
return nil
}
// Put stores the given output in the cache as the output for the action ID.
// It may read file twice. The content of file must not change between the two passes.
func (c *Cache) Put(id ActionID, file io.ReadSeeker) (OutputID, int64, error) {
return c.put(id, file, true)
}
// PutNoVerify is like Put but disables the verify check
// when GODEBUG=goverifycache=1 is set.
// It is meant for data that is OK to cache but that we expect to vary slightly from run to run,
// like test output containing times and the like.
func (c *Cache) PutNoVerify(id ActionID, file io.ReadSeeker) (OutputID, int64, error) {
return c.put(id, file, false)
}
func (c *Cache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (OutputID, int64, error) {
// Compute output ID.
h := sha256.New()
if _, err := file.Seek(0, 0); err != nil {
return OutputID{}, 0, err
}
size, err := io.Copy(h, file)
if err != nil {
return OutputID{}, 0, err
}
var out OutputID
h.Sum(out[:0])
// Copy to cached output file (if not already present).
if err := c.copyFile(file, out, size); err != nil {
return out, size, err
}
// Add to cache index.
return out, size, c.putIndexEntry(id, out, size, allowVerify)
}
// PutBytes stores the given bytes in the cache as the output for the action ID.
func (c *Cache) PutBytes(id ActionID, data []byte) error {
_, _, err := c.Put(id, bytes.NewReader(data))
return err
}
// copyFile copies file into the cache, expecting it to have the given
// output ID and size, if that file is not present already.
func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error {
name := c.fileName(out, "d")
info, err := os.Stat(name)
if err == nil && info.Size() == size {
// Check hash.
if f, err := os.Open(name); err == nil {
h := sha256.New()
io.Copy(h, f)
f.Close()
var out2 OutputID
h.Sum(out2[:0])
if out == out2 {
return nil
}
}
// Hash did not match. Fall through and rewrite file.
}
// Copy file to cache directory.
mode := os.O_RDWR | os.O_CREATE
if err == nil && info.Size() > size { // shouldn't happen but fix in case
mode |= os.O_TRUNC
}
f, err := os.OpenFile(name, mode, 0666)
if err != nil {
return err
}
defer f.Close()
if size == 0 {
// File now exists with correct size.
// Only one possible zero-length file, so contents are OK too.
// Early return here makes sure there's a "last byte" for code below.
return nil
}
// From here on, if any of the I/O writing the file fails,
// we make a best-effort attempt to truncate the file f
// before returning, to avoid leaving bad bytes in the file.
// Copy file to f, but also into h to double-check hash.
if _, err := file.Seek(0, 0); err != nil {
f.Truncate(0)
return err
}
h := sha256.New()
w := io.MultiWriter(f, h)
if _, err := io.CopyN(w, file, size-1); err != nil {
f.Truncate(0)
return err
}
// Check last byte before writing it; writing it will make the size match
// what other processes expect to find and might cause them to start
// using the file.
buf := make([]byte, 1)
if _, err := file.Read(buf); err != nil {
f.Truncate(0)
return err
}
h.Write(buf)
sum := h.Sum(nil)
if !bytes.Equal(sum, out[:]) {
f.Truncate(0)
return fmt.Errorf("file content changed underfoot")
}
// Commit cache file entry.
if _, err := f.Write(buf); err != nil {
f.Truncate(0)
return err
}
if err := f.Close(); err != nil {
// Data might not have been written,
// but file may look like it is the right size.
// To be extra careful, remove cached file.
os.Remove(name)
return err
}
os.Chtimes(name, c.now(), c.now()) // mainly for tests
return nil
}

319
libgo/go/cmd/go/internal/cache/cache_test.go vendored Normal file
View file

@ -0,0 +1,319 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cache
import (
"bytes"
"encoding/binary"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
)
func init() {
verify = false // even if GODEBUG is set
}
func TestBasic(t *testing.T) {
dir, err := ioutil.TempDir("", "cachetest-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
_, err = Open(filepath.Join(dir, "notexist"))
if err == nil {
t.Fatal(`Open("tmp/notexist") succeeded, want failure`)
}
cdir := filepath.Join(dir, "c1")
if err := os.Mkdir(cdir, 0777); err != nil {
t.Fatal(err)
}
c1, err := Open(cdir)
if err != nil {
t.Fatalf("Open(c1) (create): %v", err)
}
if err := c1.putIndexEntry(dummyID(1), dummyID(12), 13, true); err != nil {
t.Fatalf("addIndexEntry: %v", err)
}
if err := c1.putIndexEntry(dummyID(1), dummyID(2), 3, true); err != nil { // overwrite entry
t.Fatalf("addIndexEntry: %v", err)
}
if entry, err := c1.Get(dummyID(1)); err != nil || entry.OutputID != dummyID(2) || entry.Size != 3 {
t.Fatalf("c1.Get(1) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(2), 3)
}
c2, err := Open(cdir)
if err != nil {
t.Fatalf("Open(c2) (reuse): %v", err)
}
if entry, err := c2.Get(dummyID(1)); err != nil || entry.OutputID != dummyID(2) || entry.Size != 3 {
t.Fatalf("c2.Get(1) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(2), 3)
}
if err := c2.putIndexEntry(dummyID(2), dummyID(3), 4, true); err != nil {
t.Fatalf("addIndexEntry: %v", err)
}
if entry, err := c1.Get(dummyID(2)); err != nil || entry.OutputID != dummyID(3) || entry.Size != 4 {
t.Fatalf("c1.Get(2) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(3), 4)
}
}
func TestGrowth(t *testing.T) {
dir, err := ioutil.TempDir("", "cachetest-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
c, err := Open(dir)
if err != nil {
t.Fatalf("Open: %v", err)
}
n := 10000
if testing.Short() {
n = 1000
}
for i := 0; i < n; i++ {
if err := c.putIndexEntry(dummyID(i), dummyID(i*99), int64(i)*101, true); err != nil {
t.Fatalf("addIndexEntry: %v", err)
}
id := ActionID(dummyID(i))
entry, err := c.Get(id)
if err != nil {
t.Fatalf("Get(%x): %v", id, err)
}
if entry.OutputID != dummyID(i*99) || entry.Size != int64(i)*101 {
t.Errorf("Get(%x) = %x, %d, want %x, %d", id, entry.OutputID, entry.Size, dummyID(i*99), int64(i)*101)
}
}
for i := 0; i < n; i++ {
id := ActionID(dummyID(i))
entry, err := c.Get(id)
if err != nil {
t.Fatalf("Get2(%x): %v", id, err)
}
if entry.OutputID != dummyID(i*99) || entry.Size != int64(i)*101 {
t.Errorf("Get2(%x) = %x, %d, want %x, %d", id, entry.OutputID, entry.Size, dummyID(i*99), int64(i)*101)
}
}
}
func TestVerifyPanic(t *testing.T) {
os.Setenv("GODEBUG", "gocacheverify=1")
initEnv()
defer func() {
os.Unsetenv("GODEBUG")
verify = false
}()
if !verify {
t.Fatal("initEnv did not set verify")
}
dir, err := ioutil.TempDir("", "cachetest-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
c, err := Open(dir)
if err != nil {
t.Fatalf("Open: %v", err)
}
id := ActionID(dummyID(1))
if err := c.PutBytes(id, []byte("abc")); err != nil {
t.Fatal(err)
}
defer func() {
if err := recover(); err != nil {
t.Log(err)
return
}
}()
c.PutBytes(id, []byte("def"))
t.Fatal("mismatched Put did not panic in verify mode")
}
func TestCacheLog(t *testing.T) {
dir, err := ioutil.TempDir("", "cachetest-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
c, err := Open(dir)
if err != nil {
t.Fatalf("Open: %v", err)
}
c.now = func() time.Time { return time.Unix(1e9, 0) }
id := ActionID(dummyID(1))
c.Get(id)
c.PutBytes(id, []byte("abc"))
c.Get(id)
c, err = Open(dir)
if err != nil {
t.Fatalf("Open #2: %v", err)
}
c.now = func() time.Time { return time.Unix(1e9+1, 0) }
c.Get(id)
id2 := ActionID(dummyID(2))
c.Get(id2)
c.PutBytes(id2, []byte("abc"))
c.Get(id2)
c.Get(id)
data, err := ioutil.ReadFile(filepath.Join(dir, "log.txt"))
if err != nil {
t.Fatal(err)
}
want := `1000000000 miss 0100000000000000000000000000000000000000000000000000000000000000
1000000000 put 0100000000000000000000000000000000000000000000000000000000000000 ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad 3
1000000000 get 0100000000000000000000000000000000000000000000000000000000000000
1000000001 get 0100000000000000000000000000000000000000000000000000000000000000
1000000001 miss 0200000000000000000000000000000000000000000000000000000000000000
1000000001 put 0200000000000000000000000000000000000000000000000000000000000000 ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad 3
1000000001 get 0200000000000000000000000000000000000000000000000000000000000000
1000000001 get 0100000000000000000000000000000000000000000000000000000000000000
`
if string(data) != want {
t.Fatalf("log:\n%s\nwant:\n%s", string(data), want)
}
}
func dummyID(x int) [HashSize]byte {
var out [HashSize]byte
binary.LittleEndian.PutUint64(out[:], uint64(x))
return out
}
func TestCacheTrim(t *testing.T) {
dir, err := ioutil.TempDir("", "cachetest-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
c, err := Open(dir)
if err != nil {
t.Fatalf("Open: %v", err)
}
const start = 1000000000
now := int64(start)
c.now = func() time.Time { return time.Unix(now, 0) }
checkTime := func(name string, mtime int64) {
t.Helper()
file := filepath.Join(c.dir, name[:2], name)
info, err := os.Stat(file)
if err != nil {
t.Fatal(err)
}
if info.ModTime().Unix() != mtime {
t.Fatalf("%s mtime = %d, want %d", name, info.ModTime().Unix(), mtime)
}
}
id := ActionID(dummyID(1))
c.PutBytes(id, []byte("abc"))
entry, _ := c.Get(id)
c.PutBytes(ActionID(dummyID(2)), []byte("def"))
mtime := now
checkTime(fmt.Sprintf("%x-a", id), mtime)
checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime)
// Get should not change recent mtimes.
now = start + 10
c.Get(id)
checkTime(fmt.Sprintf("%x-a", id), mtime)
checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime)
// Get should change distant mtimes.
now = start + 5000
mtime2 := now
if _, err := c.Get(id); err != nil {
t.Fatal(err)
}
c.OutputFile(entry.OutputID)
checkTime(fmt.Sprintf("%x-a", id), mtime2)
checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime2)
// Trim should leave everything alone: it's all too new.
c.Trim()
if _, err := c.Get(id); err != nil {
t.Fatal(err)
}
c.OutputFile(entry.OutputID)
data, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt"))
if err != nil {
t.Fatal(err)
}
checkTime(fmt.Sprintf("%x-a", dummyID(2)), start)
// Trim less than a day later should not do any work at all.
now = start + 80000
c.Trim()
if _, err := c.Get(id); err != nil {
t.Fatal(err)
}
c.OutputFile(entry.OutputID)
data2, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt"))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, data2) {
t.Fatalf("second trim did work: %q -> %q", data, data2)
}
// Fast forward and do another trim just before the 5 day cutoff.
// Note that because of usedQuantum the cutoff is actually 5 days + 1 hour.
// We used c.Get(id) just now, so 5 days later it should still be kept.
// On the other hand almost a full day has gone by since we wrote dummyID(2)
// and we haven't looked at it since, so 5 days later it should be gone.
now += 5 * 86400
checkTime(fmt.Sprintf("%x-a", dummyID(2)), start)
c.Trim()
if _, err := c.Get(id); err != nil {
t.Fatal(err)
}
c.OutputFile(entry.OutputID)
mtime3 := now
if _, err := c.Get(dummyID(2)); err == nil { // haven't done a Get for this since original write above
t.Fatalf("Trim did not remove dummyID(2)")
}
// The c.Get(id) refreshed id's mtime again.
// Check that another 5 days later it is still not gone,
// but check by using checkTime, which doesn't bring mtime forward.
now += 5 * 86400
c.Trim()
checkTime(fmt.Sprintf("%x-a", id), mtime3)
checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3)
// Half a day later Trim should still be a no-op, because there was a Trim recently.
// Even though the entry for id is now old enough to be trimmed,
// it gets a reprieve until the time comes for a new Trim scan.
now += 86400 / 2
c.Trim()
checkTime(fmt.Sprintf("%x-a", id), mtime3)
checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3)
// Another half a day later, Trim should actually run, and it should remove id.
now += 86400/2 + 1
c.Trim()
if _, err := c.Get(dummyID(1)); err == nil {
t.Fatal("Trim did not remove dummyID(1)")
}
}

100
libgo/go/cmd/go/internal/cache/default.go vendored Normal file
View file

@ -0,0 +1,100 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cache
import (
"cmd/go/internal/base"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"sync"
)
// Default returns the default cache to use, or nil if no cache should be used.
func Default() *Cache {
defaultOnce.Do(initDefaultCache)
return defaultCache
}
var (
defaultOnce sync.Once
defaultCache *Cache
)
// cacheREADME is a message stored in a README in the cache directory.
// Because the cache lives outside the normal Go trees, we leave the
// README as a courtesy to explain where it came from.
const cacheREADME = `This directory holds cached build artifacts from the Go build system.
Run "go clean -cache" if the directory is getting too large.
See golang.org to learn more about Go.
`
// initDefaultCache does the work of finding the default cache
// the first time Default is called.
func initDefaultCache() {
dir := DefaultDir()
if dir == "off" {
return
}
if err := os.MkdirAll(dir, 0777); err != nil {
base.Fatalf("initializing cache in $GOCACHE: %s", err)
}
if _, err := os.Stat(filepath.Join(dir, "README")); err != nil {
// Best effort.
ioutil.WriteFile(filepath.Join(dir, "README"), []byte(cacheREADME), 0666)
}
c, err := Open(dir)
if err != nil {
base.Fatalf("initializing cache in $GOCACHE: %s", err)
}
defaultCache = c
}
// DefaultDir returns the effective GOCACHE setting.
// It returns "off" if the cache is disabled.
func DefaultDir() string {
dir := os.Getenv("GOCACHE")
if dir != "" {
return dir
}
// Compute default location.
// TODO(rsc): This code belongs somewhere else,
// like maybe ioutil.CacheDir or os.CacheDir.
switch runtime.GOOS {
case "windows":
dir = os.Getenv("LocalAppData")
case "darwin":
dir = os.Getenv("HOME")
if dir == "" {
return "off"
}
dir += "/Library/Caches"
case "plan9":
dir = os.Getenv("home")
if dir == "" {
return "off"
}
// Plan 9 has no established per-user cache directory,
// but $home/lib/xyz is the usual equivalent of $HOME/.xyz on Unix.
dir += "/lib/cache"
default: // Unix
// https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
dir = os.Getenv("XDG_CACHE_HOME")
if dir == "" {
dir = os.Getenv("HOME")
if dir == "" {
return "off"
}
dir += "/.cache"
}
}
return filepath.Join(dir, "go-build")
}

174
libgo/go/cmd/go/internal/cache/hash.go vendored Normal file
View file

@ -0,0 +1,174 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cache
import (
"bytes"
"crypto/sha256"
"fmt"
"hash"
"io"
"os"
"runtime"
"sync"
)
var debugHash = false // set when GODEBUG=gocachehash=1
// HashSize is the number of bytes in a hash.
const HashSize = 32
// A Hash provides access to the canonical hash function used to index the cache.
// The current implementation uses salted SHA256, but clients must not assume this.
type Hash struct {
h hash.Hash
name string // for debugging
buf *bytes.Buffer // for verify
}
// hashSalt is a salt string added to the beginning of every hash
// created by NewHash. Using the Go version makes sure that different
// versions of the go command (or even different Git commits during
// work on the development branch) do not address the same cache
// entries, so that a bug in one version does not affect the execution
// of other versions. This salt will result in additional ActionID files
// in the cache, but not additional copies of the large output files,
// which are still addressed by unsalted SHA256.
var hashSalt = []byte(runtime.Version())
// Subkey returns an action ID corresponding to mixing a parent
// action ID with a string description of the subkey.
func Subkey(parent ActionID, desc string) ActionID {
h := sha256.New()
h.Write([]byte("subkey:"))
h.Write(parent[:])
h.Write([]byte(desc))
var out ActionID
h.Sum(out[:0])
if debugHash {
fmt.Fprintf(os.Stderr, "HASH subkey %x %q = %x\n", parent, desc, out)
}
if verify {
hashDebug.Lock()
hashDebug.m[out] = fmt.Sprintf("subkey %x %q", parent, desc)
hashDebug.Unlock()
}
return out
}
// NewHash returns a new Hash.
// The caller is expected to Write data to it and then call Sum.
func NewHash(name string) *Hash {
h := &Hash{h: sha256.New(), name: name}
if debugHash {
fmt.Fprintf(os.Stderr, "HASH[%s]\n", h.name)
}
h.Write(hashSalt)
if verify {
h.buf = new(bytes.Buffer)
}
return h
}
// Write writes data to the running hash.
func (h *Hash) Write(b []byte) (int, error) {
if debugHash {
fmt.Fprintf(os.Stderr, "HASH[%s]: %q\n", h.name, b)
}
if h.buf != nil {
h.buf.Write(b)
}
return h.h.Write(b)
}
// Sum returns the hash of the data written previously.
func (h *Hash) Sum() [HashSize]byte {
var out [HashSize]byte
h.h.Sum(out[:0])
if debugHash {
fmt.Fprintf(os.Stderr, "HASH[%s]: %x\n", h.name, out)
}
if h.buf != nil {
hashDebug.Lock()
if hashDebug.m == nil {
hashDebug.m = make(map[[HashSize]byte]string)
}
hashDebug.m[out] = h.buf.String()
hashDebug.Unlock()
}
return out
}
// In GODEBUG=gocacheverify=1 mode,
// hashDebug holds the input to every computed hash ID,
// so that we can work backward from the ID involved in a
// cache entry mismatch to a description of what should be there.
var hashDebug struct {
sync.Mutex
m map[[HashSize]byte]string
}
// reverseHash returns the input used to compute the hash id.
func reverseHash(id [HashSize]byte) string {
hashDebug.Lock()
s := hashDebug.m[id]
hashDebug.Unlock()
return s
}
var hashFileCache struct {
sync.Mutex
m map[string][HashSize]byte
}
// HashFile returns the hash of the named file.
// It caches repeated lookups for a given file,
// and the cache entry for a file can be initialized
// using SetFileHash.
// The hash used by FileHash is not the same as
// the hash used by NewHash.
func FileHash(file string) ([HashSize]byte, error) {
hashFileCache.Lock()
out, ok := hashFileCache.m[file]
hashFileCache.Unlock()
if ok {
return out, nil
}
h := sha256.New()
f, err := os.Open(file)
if err != nil {
if debugHash {
fmt.Fprintf(os.Stderr, "HASH %s: %v\n", file, err)
}
return [HashSize]byte{}, err
}
_, err = io.Copy(h, f)
f.Close()
if err != nil {
if debugHash {
fmt.Fprintf(os.Stderr, "HASH %s: %v\n", file, err)
}
return [HashSize]byte{}, err
}
h.Sum(out[:0])
if debugHash {
fmt.Fprintf(os.Stderr, "HASH %s: %x\n", file, out)
}
SetFileHash(file, out)
return out, nil
}
// SetFileHash sets the hash returned by FileHash for file.
func SetFileHash(file string, sum [HashSize]byte) {
hashFileCache.Lock()
if hashFileCache.m == nil {
hashFileCache.m = make(map[string][HashSize]byte)
}
hashFileCache.m[file] = sum
hashFileCache.Unlock()
}

Some files were not shown because too many files have changed in this diff Show more