456 files changed:

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[moved from libgo/go/syscall/passfd_test.go with 62% similarity]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[moved from libgo/go/hash/crc32/crc32_amd64.go with 96% similarity]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

[new file with mode: 0644]

b/libgo/MERGE diff --git a/libgo/MERGE (file)

index 23732d0 .. b7e1968 100644 --- a/ libgo/MERGE +++ b/ libgo/MERGE @@ -1,4 +1,4 @@ -63484e8b6b76 +9895f9e36435 The first line of this file holds the Mercurial revision number of the last merge done from the master library sources.

b/libgo/go/archive/tar/common.go diff --git a/libgo/go/archive/tar/common.go (file)

index e8b973c .. e363aa7 100644 --- a/ libgo/go/archive/tar/common.go +++ b/ libgo/go/archive/tar/common.go @@ -38,6 +38,7 @@ const ( TypeXGlobalHeader = 'g' // global extended header TypeGNULongName = 'L' // Next file has a long name TypeGNULongLink = 'K' // Next file symlinks to a file w/ a long name + TypeGNUSparse = 'S' // sparse file ) // A Header represents a single header in a tar archive.

b/libgo/go/archive/tar/reader.go diff --git a/libgo/go/archive/tar/reader.go (file)

index 7cb6e64 .. 920a9b0 100644 --- a/ libgo/go/archive/tar/reader.go +++ b/ libgo/go/archive/tar/reader.go @@ -29,12 +29,57 @@ const maxNanoSecondIntSize = 9 // The Next method advances to the next file in the archive (including the first), // and then it can be treated as an io.Reader to access the file's data. type Reader struct { - r io.Reader - err error - nb int64 // number of unread bytes fo r current file entry - pad int64 // amount of padding (ignored) afte r current file entry + r io.Reader + err error + pad int64 // amount of padding (ignored) afte r current file entry + curr numBytesReader // reader fo r current file entry } +// A numBytesReader is an io.Reader with a numBytes method, returning the number +// of bytes remaining in the underlying encoded data. +type numBytesReader interface { + io.Reader + numBytes() int64 +} + +// A regFileReader is a numBytesReader for reading file data from a tar archive. +type regFileReader struct { + r io.Reader // underlying reader + nb int64 // number of unread bytes for current file entry +} + +// A sparseFileReader is a numBytesReader for reading sparse file data from a tar archive. +type sparseFileReader struct { + rfr *regFileReader // reads the sparse-encoded file data + sp []sparseEntry // the sparse map for the file + pos int64 // keeps track of file position + tot int64 // total size of the file +} + +// Keywords for GNU sparse files in a PAX extended header +const ( + paxGNUSparseNumBlocks = "GNU.sparse.numblocks" + paxGNUSparseOffset = "GNU.sparse.offset" + paxGNUSparseNumBytes = "GNU.sparse.numbytes" + paxGNUSparseMap = "GNU.sparse.map" + paxGNUSparseName = "GNU.sparse.name" + paxGNUSparseMajor = "GNU.sparse.major" + paxGNUSparseMinor = "GNU.sparse.minor" + paxGNUSparseSize = "GNU.sparse.size" + paxGNUSparseRealSize = "GNU.sparse.realsize" +) + +// Keywords for old GNU sparse headers +const ( + oldGNUSparseMainHeaderOffset = 386 + oldGNUSparseMainHeaderIsExtendedOffset = 482 + oldGNUSparseMainHeaderNumEntries = 4 + oldGNUSparseExtendedHeaderIsExtendedOffset = 504 + oldGNUSparseExtendedHeaderNumEntries = 21 + oldGNUSparseOffsetSize = 12 + oldGNUSparseNumBytesSize = 12 +) + // NewReader creates a new Reader reading from r. func NewReader(r io.Reader) *Reader { return &Reader{r: r} } @@ -64,6 +109,18 @@ func (tr *Reader) Next() (*Header, error) { tr.skipUnread() hdr = tr.readHeader() mergePAX(hdr, headers) + + // Check for a PAX format sparse file + sp, err := tr.checkForGNUSparsePAXHeaders(hdr, headers) + if err != nil { + tr.err = err + return nil, err + } + if sp != nil { + // Current file is a PAX format GNU sparse file. + // Set the current file reader to a sparse file reader. + tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size} + } return hdr, nil case TypeGNULongName: // We have a GNU long name header. Its contents are the real file name. @@ -87,6 +144,67 @@ func (tr *Reader) Next() (*Header, error) { return hdr, tr.err } +// checkForGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. If they are found, then +// this function reads the sparse map and returns it. Unknown sparse formats are ignored, causing the file to +// be treated as a regular file. +func (tr *Reader) checkForGNUSparsePAXHeaders(hdr *Header, headers map[string]string) ([]sparseEntry, error) { + var sparseFormat string + + // Check for sparse format indicators + major, majorOk := headers[paxGNUSparseMajor] + minor, minorOk := headers[paxGNUSparseMinor] + sparseName, sparseNameOk := headers[paxGNUSparseName] + _, sparseMapOk := headers[paxGNUSparseMap] + sparseSize, sparseSizeOk := headers[paxGNUSparseSize] + sparseRealSize, sparseRealSizeOk := headers[paxGNUSparseRealSize] + + // Identify which, if any, sparse format applies from which PAX headers are set + if majorOk && minorOk { + sparseFormat = major + "." + minor + } else if sparseNameOk && sparseMapOk { + sparseFormat = "0.1" + } else if sparseSizeOk { + sparseFormat = "0.0" + } else { + // Not a PAX format GNU sparse file. + return nil, nil + } + + // Check for unknown sparse format + if sparseFormat != "0.0" && sparseFormat != "0.1" && sparseFormat != "1.0" { + return nil, nil + } + + // Update hdr from GNU sparse PAX headers + if sparseNameOk { + hdr.Name = sparseName + } + if sparseSizeOk { + realSize, err := strconv.ParseInt(sparseSize, 10, 0) + if err != nil { + return nil, ErrHeader + } + hdr.Size = realSize + } else if sparseRealSizeOk { + realSize, err := strconv.ParseInt(sparseRealSize, 10, 0) + if err != nil { + return nil, ErrHeader + } + hdr.Size = realSize + } + + // Set up the sparse map, according to the particular sparse format in use + var sp []sparseEntry + var err error + switch sparseFormat { + case "0.0", "0.1": + sp, err = readGNUSparseMap0x1(headers) + case "1.0": + sp, err = readGNUSparseMap1x0(tr.curr) + } + return sp, err +} + // mergePAX merges well known headers according to PAX standard. // In general headers with the same name as those found // in the header struct overwrite those found in the header @@ -194,6 +312,11 @@ func parsePAX(r io.Reader) (map[string]string, error) { if err != nil { return nil, err } + + // For GNU PAX sparse format 0.0 support. + // This function transforms the sparse format 0.0 headers into sparse format 0.1 headers. + var sparseMap bytes.Buffer + headers := make(map[string]string) // Each record is constructed as // "%d %s=%s

", length, keyword, value @@ -211,7 +334,7 @@ func parsePAX(r io.Reader) (map[string]string, error) { return nil, ErrHeader } // Extract everything between the decimal and the n -1 on the - // beginning to to eat the ' ', -1 on the end to skip the newline. + // beginning to eat the ' ', -1 on the end to skip the newline. var record []byte record, buf = buf[sp+1:n-1], buf[n:] // The first equals is guaranteed to mark the end of the key. @@ -221,7 +344,21 @@ func parsePAX(r io.Reader) (map[string]string, error) { return nil, ErrHeader } key, value := record[:eq], record[eq+1:] - headers[string(key)] = string(value) + + keyStr := string(key) + if keyStr == paxGNUSparseOffset || keyStr == paxGNUSparseNumBytes { + // GNU sparse format 0.0 special key. Write to sparseMap instead of using the headers map. + sparseMap.Write(value) + sparseMap.Write([]byte{','}) + } else { + // Normal key. Set the value in the headers map. + headers[keyStr] = string(value) + } + } + if sparseMap.Len() != 0 { + // Add sparse info to headers, chopping off the extra comma + sparseMap.Truncate(sparseMap.Len() - 1) + headers[paxGNUSparseMap] = sparseMap.String() } return headers, nil } @@ -268,8 +405,8 @@ func (tr *Reader) octal(b []byte) int64 { // skipUnread skips any unread bytes in the existing file entry, as well as any alignment padding. func (tr *Reader) skipUnread() { - nr := tr.n b + tr.pad // number of bytes to skip - tr. nb, tr.pad = 0 , 0 + nr := tr.n umBytes() + tr.pad // number of bytes to skip + tr. curr, tr.pad = nil , 0 if sr, ok := tr.r.(io.Seeker); ok { if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil { return @@ -331,14 +468,14 @@ func (tr *Reader) readHeader() *Header { // so its magic bytes, like the rest of the block, are NULs. magic := string(s.next(8)) // contains version field as well. var format string - switch magic { - case "ustar\x00 00": // POSIX tar (1003.1-1988) + switch { + case magic[:6] == "ustar\x 00": // POSIX tar (1003.1-1988) if string(header[508:512]) == "tar\x00" { format = "star" } else { format = "posix" } - case "ustar \x00": // old GNU tar + case magic == "ustar \x00": // old GNU tar format = "gnu" } @@ -373,30 +510,308 @@ func (tr *Reader) readHeader() *Header { // Maximum value of hdr.Size is 64 GB (12 octal digits), // so there's no risk of int64 overflowing. - tr.nb = int64(hdr.Size) - tr.pad = -tr.nb & (blockSize - 1) // blockSize is a power of two + nb := int64(hdr.Size) + tr.pad = -nb & (blockSize - 1) // blockSize is a power of two + + // Set the current file reader. + tr.curr = ®FileReader{r: tr.r, nb: nb} + + // Check for old GNU sparse format entry. + if hdr.Typeflag == TypeGNUSparse { + // Get the real size of the file. + hdr.Size = tr.octal(header[483:495]) + + // Read the sparse map. + sp := tr.readOldGNUSparseMap(header) + if tr.err != nil { + return nil + } + // Current file is a GNU sparse file. Update the current file reader. + tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size} + } return hdr } +// A sparseEntry holds a single entry in a sparse file's sparse map. +// A sparse entry indicates the offset and size in a sparse file of a +// block of data. +type sparseEntry struct { + offset int64 + numBytes int64 +} + +// readOldGNUSparseMap reads the sparse map as stored in the old GNU sparse format. +// The sparse map is stored in the tar header if it's small enough. If it's larger than four entries, +// then one or more extension headers are used to store the rest of the sparse map. +func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry { + isExtended := header[oldGNUSparseMainHeaderIsExtendedOffset] != 0 + spCap := oldGNUSparseMainHeaderNumEntries + if isExtended { + spCap += oldGNUSparseExtendedHeaderNumEntries + } + sp := make([]sparseEntry, 0, spCap) + s := slicer(header[oldGNUSparseMainHeaderOffset:]) + + // Read the four entries from the main tar header + for i := 0; i < oldGNUSparseMainHeaderNumEntries; i++ { + offset := tr.octal(s.next(oldGNUSparseOffsetSize)) + numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize)) + if tr.err != nil { + tr.err = ErrHeader + return nil + } + if offset == 0 && numBytes == 0 { + break + } + sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) + } + + for isExtended { + // There are more entries. Read an extension header and parse its entries. + sparseHeader := make([]byte, blockSize) + if _, tr.err = io.ReadFull(tr.r, sparseHeader); tr.err != nil { + return nil + } + isExtended = sparseHeader[oldGNUSparseExtendedHeaderIsExtendedOffset] != 0 + s = slicer(sparseHeader) + for i := 0; i < oldGNUSparseExtendedHeaderNumEntries; i++ { + offset := tr.octal(s.next(oldGNUSparseOffsetSize)) + numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize)) + if tr.err != nil { + tr.err = ErrHeader + return nil + } + if offset == 0 && numBytes == 0 { + break + } + sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) + } + } + return sp +} + +// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format version 1.0. +// The sparse map is stored just before the file data and padded out to the nearest block boundary. +func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) { + buf := make([]byte, 2*blockSize) + sparseHeader := buf[:blockSize] + + // readDecimal is a helper function to read a decimal integer from the sparse map + // while making sure to read from the file in blocks of size blockSize + readDecimal := func() (int64, error) { + // Look for newline + nl := bytes.IndexByte(sparseHeader, '

') + if nl == -1 { + if len(sparseHeader) >= blockSize { + // This is an error + return 0, ErrHeader + } + oldLen := len(sparseHeader) + newLen := oldLen + blockSize + if cap(sparseHeader) < newLen { + // There's more header, but we need to make room for the next block + copy(buf, sparseHeader) + sparseHeader = buf[:newLen] + } else { + // There's more header, and we can just reslice + sparseHeader = sparseHeader[:newLen] + } + + // Now that sparseHeader is large enough, read next block + if _, err := io.ReadFull(r, sparseHeader[oldLen:newLen]); err != nil { + return 0, err + } + + // Look for a newline in the new data + nl = bytes.IndexByte(sparseHeader[oldLen:newLen], '

') + if nl == -1 { + // This is an error + return 0, ErrHeader + } + nl += oldLen // We want the position from the beginning + } + // Now that we've found a newline, read a number + n, err := strconv.ParseInt(string(sparseHeader[:nl]), 10, 0) + if err != nil { + return 0, ErrHeader + } + + // Update sparseHeader to consume this number + sparseHeader = sparseHeader[nl+1:] + return n, nil + } + + // Read the first block + if _, err := io.ReadFull(r, sparseHeader); err != nil { + return nil, err + } + + // The first line contains the number of entries + numEntries, err := readDecimal() + if err != nil { + return nil, err + } + + // Read all the entries + sp := make([]sparseEntry, 0, numEntries) + for i := int64(0); i < numEntries; i++ { + // Read the offset + offset, err := readDecimal() + if err != nil { + return nil, err + } + // Read numBytes + numBytes, err := readDecimal() + if err != nil { + return nil, err + } + + sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) + } + + return sp, nil +} + +// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format version 0.1. +// The sparse map is stored in the PAX headers. +func readGNUSparseMap0x1(headers map[string]string) ([]sparseEntry, error) { + // Get number of entries + numEntriesStr, ok := headers[paxGNUSparseNumBlocks] + if !ok { + return nil, ErrHeader + } + numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) + if err != nil { + return nil, ErrHeader + } + + sparseMap := strings.Split(headers[paxGNUSparseMap], ",") + + // There should be two numbers in sparseMap for each entry + if int64(len(sparseMap)) != 2*numEntries { + return nil, ErrHeader + } + + // Loop through the entries in the sparse map + sp := make([]sparseEntry, 0, numEntries) + for i := int64(0); i < numEntries; i++ { + offset, err := strconv.ParseInt(sparseMap[2*i], 10, 0) + if err != nil { + return nil, ErrHeader + } + numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 0) + if err != nil { + return nil, ErrHeader + } + sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) + } + + return sp, nil +} + +// numBytes returns the number of bytes left to read in the current file's entry +// in the tar archive, or 0 if there is no current file. +func (tr *Reader) numBytes() int64 { + if tr.curr == nil { + // No current file, so no bytes + return 0 + } + return tr.curr.numBytes() +} + // Read reads from the current entry in the tar archive. // It returns 0, io.EOF when it reaches the end of that entry, // until Next is called to advance to the next entry. func (tr *Reader) Read(b []byte) (n int, err error) { - if tr.nb == 0 { - // file consumed + if tr.curr == nil { return 0, io.EOF } + n, err = tr.curr.Read(b) + if err != nil && err != io.EOF { + tr.err = err + } + return +} - if int64(len(b)) > tr.nb { - b = b[0:tr.nb] +func (rfr *regFileReader) Read(b []byte) (n int, err error) { + if rfr.nb == 0 { + // file consumed + return 0, io.EOF } - n, err = tr.r.Read(b) - tr.nb -= int64(n) + if int64(len(b)) > rfr.nb { + b = b[0:rfr.nb] + } + n, err = rfr.r.Read(b) + rfr.nb -= int64(n) - if err == io.EOF && t r.nb > 0 { + if err == io.EOF && rf r.nb > 0 { err = io.ErrUnexpectedEOF } - tr.err = err return } + +// numBytes returns the number of bytes left to read in the file's data in the tar archive. +func (rfr *regFileReader) numBytes() int64 { + return rfr.nb +} + +// readHole reads a sparse file hole ending at offset toOffset +func (sfr *sparseFileReader) readHole(b []byte, toOffset int64) int { + n64 := toOffset - sfr.pos + if n64 > int64(len(b)) { + n64 = int64(len(b)) + } + n := int(n64) + for i := 0; i < n; i++ { + b[i] = 0 + } + sfr.pos += n64 + return n +} + +// Read reads the sparse file data in expanded form. +func (sfr *sparseFileReader) Read(b []byte) (n int, err error) { + if len(sfr.sp) == 0 { + // No more data fragments to read from. + if sfr.pos < sfr.tot { + // We're in the last hole + n = sfr.readHole(b, sfr.tot) + return + } + // Otherwise, we're at the end of the file + return 0, io.EOF + } + if sfr.pos < sfr.sp[0].offset { + // We're in a hole + n = sfr.readHole(b, sfr.sp[0].offset) + return + } + + // We're not in a hole, so we'll read from the next data fragment + posInFragment := sfr.pos - sfr.sp[0].offset + bytesLeft := sfr.sp[0].numBytes - posInFragment + if int64(len(b)) > bytesLeft { + b = b[0:bytesLeft] + } + + n, err = sfr.rfr.Read(b) + sfr.pos += int64(n) + + if int64(n) == bytesLeft { + // We're done with this fragment + sfr.sp = sfr.sp[1:] + } + + if err == io.EOF && sfr.pos < sfr.tot { + // We reached the end of the last fragment's data, but there's a final hole + err = nil + } + return +} + +// numBytes returns the number of bytes left to read in the sparse file's +// sparse-encoded data in the tar archive. +func (sfr *sparseFileReader) numBytes() int64 { + return sfr.rfr.nb +}

b/libgo/go/archive/tar/reader_test.go diff --git a/libgo/go/archive/tar/reader_test.go (file)

index f84dbeb .. 9601ffe 100644 --- a/ libgo/go/archive/tar/reader_test.go +++ b/ libgo/go/archive/tar/reader_test.go @@ -9,6 +9,7 @@ import ( "crypto/md5" "fmt" "io" + "io/ioutil" "os" "reflect" "strings" @@ -54,8 +55,92 @@ var gnuTarTest = &untarTest{ }, } +var sparseTarTest = &untarTest{ + file: "testdata/sparse-formats.tar", + headers: []*Header{ + { + Name: "sparse-gnu", + Mode: 420, + Uid: 1000, + Gid: 1000, + Size: 200, + ModTime: time.Unix(1392395740, 0), + Typeflag: 0x53, + Linkname: "", + Uname: "david", + Gname: "david", + Devmajor: 0, + Devminor: 0, + }, + { + Name: "sparse-posix-0.0", + Mode: 420, + Uid: 1000, + Gid: 1000, + Size: 200, + ModTime: time.Unix(1392342187, 0), + Typeflag: 0x30, + Linkname: "", + Uname: "david", + Gname: "david", + Devmajor: 0, + Devminor: 0, + }, + { + Name: "sparse-posix-0.1", + Mode: 420, + Uid: 1000, + Gid: 1000, + Size: 200, + ModTime: time.Unix(1392340456, 0), + Typeflag: 0x30, + Linkname: "", + Uname: "david", + Gname: "david", + Devmajor: 0, + Devminor: 0, + }, + { + Name: "sparse-posix-1.0", + Mode: 420, + Uid: 1000, + Gid: 1000, + Size: 200, + ModTime: time.Unix(1392337404, 0), + Typeflag: 0x30, + Linkname: "", + Uname: "david", + Gname: "david", + Devmajor: 0, + Devminor: 0, + }, + { + Name: "end", + Mode: 420, + Uid: 1000, + Gid: 1000, + Size: 4, + ModTime: time.Unix(1392398319, 0), + Typeflag: 0x30, + Linkname: "", + Uname: "david", + Gname: "david", + Devmajor: 0, + Devminor: 0, + }, + }, + cksums: []string{ + "6f53234398c2449fe67c1812d993012f", + "6f53234398c2449fe67c1812d993012f", + "6f53234398c2449fe67c1812d993012f", + "6f53234398c2449fe67c1812d993012f", + "b0061974914468de549a2af8ced10316", + }, +} + var untarTests = []*untarTest{ gnuTarTest, + sparseTarTest, { file: "testdata/star.tar", headers: []*Header{ @@ -386,7 +471,7 @@ func TestParsePAXHeader(t *testing.T) { func TestParsePAXTime(t *testing.T) { // Some valid PAX time values timestamps := map[string]time.Time{ - "1350244992.023960108": time.Unix(1350244992, 23960108), // The commo o n case + "1350244992.023960108": time.Unix(1350244992, 23960108), // The common case "1350244992.02396010": time.Unix(1350244992, 23960100), // Lower precision value "1350244992.0239601089": time.Unix(1350244992, 23960108), // Higher precision value "1350244992": time.Unix(1350244992, 0), // Low precision value @@ -423,3 +508,236 @@ func TestMergePAX(t *testing.T) { t.Errorf("incorrect merge: got %+v, want %+v", hdr, want) } } + +func TestSparseEndToEnd(t *testing.T) { + test := sparseTarTest + f, err := os.Open(test.file) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer f.Close() + + tr := NewReader(f) + + headers := test.headers + cksums := test.cksums + nread := 0 + + // loop over all files + for ; ; nread++ { + hdr, err := tr.Next() + if hdr == nil || err == io.EOF { + break + } + + // check the header + if !reflect.DeepEqual(*hdr, *headers[nread]) { + t.Errorf("Incorrect header:

have %+v

want %+v", + *hdr, headers[nread]) + } + + // read and checksum the file data + h := md5.New() + _, err = io.Copy(h, tr) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + // verify checksum + have := fmt.Sprintf("%x", h.Sum(nil)) + want := cksums[nread] + if want != have { + t.Errorf("Bad checksum on file %s:

have %+v

want %+v", hdr.Name, have, want) + } + } + if nread != len(headers) { + t.Errorf("Didn't process all files

expected: %d

processed %d

", len(headers), nread) + } +} + +type sparseFileReadTest struct { + sparseData []byte + sparseMap []sparseEntry + realSize int64 + expected []byte +} + +var sparseFileReadTests = []sparseFileReadTest{ + { + sparseData: []byte("abcde"), + sparseMap: []sparseEntry{ + {offset: 0, numBytes: 2}, + {offset: 5, numBytes: 3}, + }, + realSize: 8, + expected: []byte("ab\x00\x00\x00cde"), + }, + { + sparseData: []byte("abcde"), + sparseMap: []sparseEntry{ + {offset: 0, numBytes: 2}, + {offset: 5, numBytes: 3}, + }, + realSize: 10, + expected: []byte("ab\x00\x00\x00cde\x00\x00"), + }, + { + sparseData: []byte("abcde"), + sparseMap: []sparseEntry{ + {offset: 1, numBytes: 3}, + {offset: 6, numBytes: 2}, + }, + realSize: 8, + expected: []byte("\x00abc\x00\x00de"), + }, + { + sparseData: []byte("abcde"), + sparseMap: []sparseEntry{ + {offset: 1, numBytes: 3}, + {offset: 6, numBytes: 2}, + }, + realSize: 10, + expected: []byte("\x00abc\x00\x00de\x00\x00"), + }, + { + sparseData: []byte(""), + sparseMap: nil, + realSize: 2, + expected: []byte("\x00\x00"), + }, +} + +func TestSparseFileReader(t *testing.T) { + for i, test := range sparseFileReadTests { + r := bytes.NewReader(test.sparseData) + nb := int64(r.Len()) + sfr := &sparseFileReader{ + rfr: ®FileReader{r: r, nb: nb}, + sp: test.sparseMap, + pos: 0, + tot: test.realSize, + } + if sfr.numBytes() != nb { + t.Errorf("test %d: Before reading, sfr.numBytes() = %d, want %d", i, sfr.numBytes(), nb) + } + buf, err := ioutil.ReadAll(sfr) + if err != nil { + t.Errorf("test %d: Unexpected error: %v", i, err) + } + if e := test.expected; !bytes.Equal(buf, e) { + t.Errorf("test %d: Contents = %v, want %v", i, buf, e) + } + if sfr.numBytes() != 0 { + t.Errorf("test %d: After draining the reader, numBytes() was nonzero", i) + } + } +} + +func TestSparseIncrementalRead(t *testing.T) { + sparseMap := []sparseEntry{{10, 2}} + sparseData := []byte("Go") + expected := "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Go\x00\x00\x00\x00\x00\x00\x00\x00" + + r := bytes.NewReader(sparseData) + nb := int64(r.Len()) + sfr := &sparseFileReader{ + rfr: ®FileReader{r: r, nb: nb}, + sp: sparseMap, + pos: 0, + tot: int64(len(expected)), + } + + // We'll read the data 6 bytes at a time, with a hole of size 10 at + // the beginning and one of size 8 at the end. + var outputBuf bytes.Buffer + buf := make([]byte, 6) + for { + n, err := sfr.Read(buf) + if err == io.EOF { + break + } + if err != nil { + t.Errorf("Read: unexpected error %v

", err) + } + if n > 0 { + _, err := outputBuf.Write(buf[:n]) + if err != nil { + t.Errorf("Write: unexpected error %v

", err) + } + } + } + got := outputBuf.String() + if got != expected { + t.Errorf("Contents = %v, want %v", got, expected) + } +} + +func TestReadGNUSparseMap0x1(t *testing.T) { + headers := map[string]string{ + paxGNUSparseNumBlocks: "4", + paxGNUSparseMap: "0,5,10,5,20,5,30,5", + } + expected := []sparseEntry{ + {offset: 0, numBytes: 5}, + {offset: 10, numBytes: 5}, + {offset: 20, numBytes: 5}, + {offset: 30, numBytes: 5}, + } + + sp, err := readGNUSparseMap0x1(headers) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if !reflect.DeepEqual(sp, expected) { + t.Errorf("Incorrect sparse map: got %v, wanted %v", sp, expected) + } +} + +func TestReadGNUSparseMap1x0(t *testing.T) { + // This test uses lots of holes so the sparse header takes up more than two blocks + numEntries := 100 + expected := make([]sparseEntry, 0, numEntries) + sparseMap := new(bytes.Buffer) + + fmt.Fprintf(sparseMap, "%d

", numEntries) + for i := 0; i < numEntries; i++ { + offset := int64(2048 * i) + numBytes := int64(1024) + expected = append(expected, sparseEntry{offset: offset, numBytes: numBytes}) + fmt.Fprintf(sparseMap, "%d

%d

", offset, numBytes) + } + + // Make the header the smallest multiple of blockSize that fits the sparseMap + headerBlocks := (sparseMap.Len() + blockSize - 1) / blockSize + bufLen := blockSize * headerBlocks + buf := make([]byte, bufLen) + copy(buf, sparseMap.Bytes()) + + // Get an reader to read the sparse map + r := bytes.NewReader(buf) + + // Read the sparse map + sp, err := readGNUSparseMap1x0(r) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if !reflect.DeepEqual(sp, expected) { + t.Errorf("Incorrect sparse map: got %v, wanted %v", sp, expected) + } +} + +func TestUninitializedRead(t *testing.T) { + test := gnuTarTest + f, err := os.Open(test.file) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer f.Close() + + tr := NewReader(f) + _, err = tr.Read([]byte{}) + if err == nil || err != io.EOF { + t.Errorf("Unexpected error: %v, wanted %v", err, io.EOF) + } + +}

diff --git a/libgo/go/archive/tar/testdata/sparse-formats.tar b/libgo/go/archive/tar/testdata/sparse-formats.tar (file)

index 0000000..

Binary files /dev/null and b/libgo/go/archive/tar/testdata/sparse-formats.tar differ

new file mode 100644index 0000000.. 8bd4e74 Binary files /dev/null and b/libgo/go/archive/tar/testdata/sparse-formats.tar differ

diff --git a/libgo/go/archive/tar/testdata/writer-big-long.tar b/libgo/go/archive/tar/testdata/writer-big-long.tar (file)

index 0000000..

Binary files /dev/null and b/libgo/go/archive/tar/testdata/writer-big-long.tar differ

new file mode 100644index 0000000.. 5960ee8 Binary files /dev/null and b/libgo/go/archive/tar/testdata/writer-big-long.tar differ

b/libgo/go/archive/tar/writer.go diff --git a/libgo/go/archive/tar/writer.go (file)

index 9ee9499 .. 6eff6f6 100644 --- a/ libgo/go/archive/tar/writer.go +++ b/ libgo/go/archive/tar/writer.go @@ -218,8 +218,8 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { tw.cString(prefixHeaderBytes, prefix, false, paxNone, nil) // Use the ustar magic if we used ustar long names. - if len(prefix) > 0 { - copy(header[257:265], []byte("ustar\ 0 00")) + if len(prefix) > 0 && !tw.usedBinary { + copy(header[257:265], []byte("ustar\ x 00")) } } }

b/libgo/go/archive/tar/writer_test.go diff --git a/libgo/go/archive/tar/writer_test.go (file)

index 2b9ea65 .. 512fab1 100644 --- a/ libgo/go/archive/tar/writer_test.go +++ b/ libgo/go/archive/tar/writer_test.go @@ -103,6 +103,29 @@ var writerTests = []*writerTest{ }, }, }, + // The truncated test file was produced using these commands: + // dd if=/dev/zero bs=1048576 count=16384 > (longname/)*15 /16gig.txt + // tar -b 1 -c -f- (longname/)*15 /16gig.txt | dd bs=512 count=8 > writer-big-long.tar + { + file: "testdata/writer-big-long.tar", + entries: []*writerTestEntry{ + { + header: &Header{ + Name: strings.Repeat("longname/", 15) + "16gig.txt", + Mode: 0644, + Uid: 1000, + Gid: 1000, + Size: 16 << 30, + ModTime: time.Unix(1399583047, 0), + Typeflag: '0', + Uname: "guillaume", + Gname: "guillaume", + }, + // fake contents + contents: strings.Repeat("\x00", 4<<10), + }, + }, + }, // This file was produced using gnu tar 1.17 // gnutar -b 4 --format=ustar (longname/)*15 + file.txt {

b/libgo/go/archive/zip/struct.go diff --git a/libgo/go/archive/zip/struct.go (file)

index 65e5238 .. cb28e83 100644 --- a/ libgo/go/archive/zip/struct.go +++ b/ libgo/go/archive/zip/struct.go @@ -174,13 +174,13 @@ func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) { return } -// ModTime returns the modification time. +// ModTime returns the modification time in UTC . // The resolution is 2s. func (h *FileHeader) ModTime() time.Time { return msDosTimeToTime(h.ModifiedDate, h.ModifiedTime) } -// SetModTime sets the ModifiedTime and ModifiedDate fields to the given time. +// SetModTime sets the ModifiedTime and ModifiedDate fields to the given time in UTC . // The resolution is 2s. func (h *FileHeader) SetModTime(t time.Time) { h.ModifiedDate, h.ModifiedTime = timeToMsDosTime(t)

b/libgo/go/bufio/bufio.go diff --git a/libgo/go/bufio/bufio.go (file)

index d1ff3c9 .. 61ef261 100644 --- a/ libgo/go/bufio/bufio.go +++ b/ libgo/go/bufio/bufio.go @@ -38,6 +38,7 @@ type Reader struct { } const minReadBufferSize = 16 +const maxConsecutiveEmptyReads = 100 // NewReaderSize returns a new Reader whose buffer has at least the specified // size. If the argument io.Reader is already a Reader with large enough @@ -87,15 +88,26 @@ func (b *Reader) fill() { b.r = 0 } - // Read new data. - n, err := b.rd.Read(b.buf[b.w:]) - if n < 0 { - panic(errNegativeRead) + if b.w >= len(b.buf) { + panic("bufio: tried to fill full buffer") } - b.w += n - if err != nil { - b.err = err + + // Read new data: try a limited number of times. + for i := maxConsecutiveEmptyReads; i > 0; i-- { + n, err := b.rd.Read(b.buf[b.w:]) + if n < 0 { + panic(errNegativeRead) + } + b.w += n + if err != nil { + b.err = err + return + } + if n > 0 { + return + } } + b.err = io.ErrNoProgress } func (b *Reader) readErr() error { @@ -115,8 +127,9 @@ func (b *Reader) Peek(n int) ([]byte, error) { if n > len(b.buf) { return nil, ErrBufferFull } + // 0 <= n <= len(b.buf) for b.w-b.r < n && b.err == nil { - b.fill() + b.fill() // b.w-b.r < len(b.buf) => buffer is not full } m := b.w - b.r if m > n { @@ -142,7 +155,7 @@ func (b *Reader) Read(p []byte) (n int, err error) { if n == 0 { return 0, b.readErr() } - if b. w == b.r { + if b. r == b.w { if b.err != nil { return 0, b.readErr() } @@ -150,13 +163,16 @@ func (b *Reader) Read(p []byte) (n int, err error) { // Large read, empty buffer. // Read directly into p to avoid copy. n, b.err = b.rd.Read(p) + if n < 0 { + panic(errNegativeRead) + } if n > 0 { b.lastByte = int(p[n-1]) b.lastRuneSize = -1 } return n, b.readErr() } - b.fill() + b.fill() // buffer is empty if b.w == b.r { return 0, b.readErr() } @@ -176,11 +192,11 @@ func (b *Reader) Read(p []byte) (n int, err error) { // If no byte is available, returns an error. func (b *Reader) ReadByte() (c byte, err error) { b.lastRuneSize = -1 - for b. w == b.r { + for b. r == b.w { if b.err != nil { return 0, b.readErr() } - b.fill() + b.fill() // buffer is empty } c = b.buf[b.r] b.r++ @@ -190,19 +206,19 @@ func (b *Reader) ReadByte() (c byte, err error) { // UnreadByte unreads the last byte. Only the most recently read byte can be unread. func (b *Reader) UnreadByte() error { - b.lastRuneSize = -1 - if b.r == b.w && b.lastByte >= 0 { - b.w = 1 - b.r = 0 - b.buf[0] = byte(b.lastByte) - b.lastByte = -1 - return nil - } - if b.r <= 0 { + if b.lastByte < 0 || b.r == 0 && b.w > 0 { return ErrInvalidUnreadByte } - b.r-- + // b.r > 0 || b.w == 0 + if b.r > 0 { + b.r-- + } else { + // b.r == 0 && b.w == 0 + b.w = 1 + } + b.buf[b.r] = byte(b.lastByte) b.lastByte = -1 + b.lastRuneSize = -1 return nil } @@ -210,8 +226,8 @@ func (b *Reader) UnreadByte() error { // rune and its size in bytes. If the encoded rune is invalid, it consumes one byte // and returns unicode.ReplacementChar (U+FFFD) with a size of 1. func (b *Reader) ReadRune() (r rune, size int, err error) { - for b.r+utf8.UTFMax > b.w && !utf8.FullRune(b.buf[b.r:b.w]) && b.err == nil { - b.fill() + for b.r+utf8.UTFMax > b.w && !utf8.FullRune(b.buf[b.r:b.w]) && b.err == nil && b.w-b.r < len(b.buf) { + b.fill() // b.w-b.r < len(buf) => buffer is not full } b.lastRuneSize = -1 if b.r == b.w { @@ -232,7 +248,7 @@ func (b *Reader) ReadRune() (r rune, size int, err error) { // regard it is stricter than UnreadByte, which will unread the last byte // from any read operation.) func (b *Reader) UnreadRune() error { - if b.lastRuneSize < 0 || b.r == 0 { + if b.lastRuneSize < 0 || b.r < b.lastRuneSize { return ErrInvalidUnreadRune } b.r -= b.lastRuneSize @@ -255,37 +271,39 @@ func (b *Reader) Buffered() int { return b.w - b.r } // ReadBytes or ReadString instead. // ReadSlice returns err != nil if and only if line does not end in delim. func (b *Reader) ReadSlice(delim byte) (line []byte, err error) { - // Look in buffer. - if i := bytes.IndexByte(b.buf[b.r:b.w], delim); i >= 0 { - line1 := b.buf[b.r : b.r+i+1] - b.r += i + 1 - return line1, nil - } - - // Read more into buffer, until buffer fills or we find delim. for { - if b.err != nil { - line := b.buf[b.r:b.w] - b.r = b.w - return line, b.readErr() + // Search buffer. + if i := bytes.IndexByte(b.buf[b.r:b.w], delim); i >= 0 { + line = b.buf[b.r : b.r+i+1] + b.r += i + 1 + break } - n := b.Buffered() - b.fill() - - // Search new part of buffer - if i := bytes.IndexByte(b.buf[n:b.w], delim); i >= 0 { - line := b.buf[0 : n+i+1] - b.r = n + i + 1 - return line, nil + // Pending error? + if b.err != nil { + line = b.buf[b.r:b.w] + b.r = b.w + err = b.readErr() + break } - // Buffer is full? - if b.Buffered() >= len(b.buf) { + // Buffer full? + if n := b.Buffered(); n >= len(b.buf) { b.r = b.w - return b.buf, ErrBufferFull + line = b.buf + err = ErrBufferFull + break } + + b.fill() // buffer is not full } + + // Handle last byte, if any. + if i := len(line) - 1; i >= 0 { + b.lastByte = int(line[i]) + } + + return } // ReadLine is a low-level line-reading primitive. Most callers should use @@ -301,6 +319,9 @@ func (b *Reader) ReadSlice(delim byte) (line []byte, err error) { // // The text returned from ReadLine does not include the line end ("\r

" or "

"). // No indication or error is given if the input ends without a final line end. +// Calling UnreadByte after ReadLine will always unread the last byte read +// (possibly a character belonging to the line end) even if that byte is not +// part of the line returned by ReadLine. func (b *Reader) ReadLine() (line []byte, isPrefix bool, err error) { line, err = b.ReadSlice('

') if err == ErrBufferFull { @@ -410,12 +431,24 @@ func (b *Reader) WriteTo(w io.Writer) (n int64, err error) { return n, err } - for b.fill(); b.r < b.w; b.fill() { + if w, ok := w.(io.ReaderFrom); ok { + m, err := w.ReadFrom(b.rd) + n += m + return n, err + } + + if b.w-b.r < len(b.buf) { + b.fill() // buffer not full + } + + for b.r < b.w { + // b.r < b.w => buffer is not empty m, err := b.writeBuf(w) n += m if err != nil { return n, err } + b.fill() // buffer is empty } if b.err == io.EOF { @@ -428,6 +461,9 @@ func (b *Reader) WriteTo(w io.Writer) (n int64, err error) { // writeBuf writes the Reader's buffer to the writer. func (b *Reader) writeBuf(w io.Writer) (int64, error) { n, err := w.Write(b.buf[b.r:b.w]) + if n < b.r-b.w { + panic(errors.New("bufio: writer did not write all data")) + } b.r += n return int64(n), err } @@ -619,9 +655,16 @@ func (b *Writer) ReadFrom(r io.Reader) (n int64, err error) { return n, err1 } } - m, err = r.Read(b.buf[b.n:]) - if m == 0 { - break + nr := 0 + for nr < maxConsecutiveEmptyReads { + m, err = r.Read(b.buf[b.n:]) + if m != 0 || err != nil { + break + } + nr++ + } + if nr == maxConsecutiveEmptyReads { + return n, io.ErrNoProgress } b.n += m n += int64(m)

b/libgo/go/bufio/bufio_test.go diff --git a/libgo/go/bufio/bufio_test.go (file)

index 3c86857 .. 76d3c8e 100644 --- a/ libgo/go/bufio/bufio_test.go +++ b/ libgo/go/bufio/bufio_test.go @@ -14,6 +14,7 @@ import ( "strings" "testing" "testing/iotest" + "time" "unicode/utf8" ) @@ -174,6 +175,34 @@ func TestReader(t *testing.T) { } } +type zeroReader struct{} + +func (zeroReader) Read(p []byte) (int, error) { + return 0, nil +} + +func TestZeroReader(t *testing.T) { + var z zeroReader + r := NewReader(z) + + c := make(chan error) + go func() { + _, err := r.ReadByte() + c <- err + }() + + select { + case err := <-c: + if err == nil { + t.Error("error expected") + } else if err != io.ErrNoProgress { + t.Error("unexpected error:", err) + } + case <-time.After(time.Second): + t.Error("test timed out (endless loop in ReadByte?)") + } +} + // A StringReader delivers its data one string segment at a time via Read. type StringReader struct { data []string @@ -228,66 +257,150 @@ func TestReadRune(t *testing.T) { } func TestUnreadRune(t *testing.T) { - got := "" segments := []string{"Hello, world:", "日本語"} - data := strings.Join(segments, "") r := NewReader(&StringReader{data: segments}) + got := "" + want := strings.Join(segments, "") // Normal execution. for { r1, _, err := r.ReadRune() if err != nil { if err != io.EOF { - t.Error("unexpected EOF" ) + t.Error("unexpected error on ReadRune:", err ) } break } got += string(r1) - // Put it back and read it again + // Put it back and read it again . if err = r.UnreadRune(); err != nil { - t. Error ("unexpected error on UnreadRune:", err) + t. Fatal ("unexpected error on UnreadRune:", err) } r2, _, err := r.ReadRune() if err != nil { - t. Error ("unexpected error reading after unreading:", err) + t. Fatal ("unexpected error reading after unreading:", err) } if r1 != r2 { - t. Errorf("incorrect rune after unread: got %c wanted %c", r1, r2) + t. Fatalf("incorrect rune after unread: got %c, want %c", r1, r2) } } - if got != data { - t.Errorf(" want=%q got=%q", data, go t) + if got != want { + t.Errorf(" got %q, want %q", got, wan t) } } func TestUnreadByte(t *testing.T) { - want := "Hello, world" - got := "" segments := []string{"Hello, ", "world"} r := NewReader(&StringReader{data: segments}) + got := "" + want := strings.Join(segments, "") // Normal execution. for { b1, err := r.ReadByte() if err != nil { if err != io.EOF { - t. Fatal("unexpected EOF" ) + t. Error("unexpected error on ReadByte:", err ) } break } got += string(b1) - // Put it back and read it again + // Put it back and read it again . if err = r.UnreadByte(); err != nil { - t.Fatal f("unexpected error on UnreadByte: %v ", err) + t.Fatal ("unexpected error on UnreadByte: ", err) } b2, err := r.ReadByte() if err != nil { - t.Fatal f("unexpected error reading after unreading: %v ", err) + t.Fatal ("unexpected error reading after unreading: ", err) } if b1 != b2 { - t.Fatalf("incorrect byte after unread: got % c wanted %c ", b1, b2) + t.Fatalf("incorrect byte after unread: got % q, want %q ", b1, b2) } } if got != want { - t.Errorf("got=%q want=%q", got, want) + t.Errorf("got %q, want %q", got, want) + } +} + +func TestUnreadByteMultiple(t *testing.T) { + segments := []string{"Hello, ", "world"} + data := strings.Join(segments, "") + for n := 0; n <= len(data); n++ { + r := NewReader(&StringReader{data: segments}) + // Read n bytes. + for i := 0; i < n; i++ { + b, err := r.ReadByte() + if err != nil { + t.Fatalf("n = %d: unexpected error on ReadByte: %v", n, err) + } + if b != data[i] { + t.Fatalf("n = %d: incorrect byte returned from ReadByte: got %q, want %q", n, b, data[i]) + } + } + // Unread one byte if there is one. + if n > 0 { + if err := r.UnreadByte(); err != nil { + t.Errorf("n = %d: unexpected error on UnreadByte: %v", n, err) + } + } + // Test that we cannot unread any further. + if err := r.UnreadByte(); err == nil { + t.Errorf("n = %d: expected error on UnreadByte", n) + } + } +} + +func TestUnreadByteOthers(t *testing.T) { + // A list of readers to use in conjunction with UnreadByte. + var readers = []func(*Reader, byte) ([]byte, error){ + (*Reader).ReadBytes, + (*Reader).ReadSlice, + func(r *Reader, delim byte) ([]byte, error) { + data, err := r.ReadString(delim) + return []byte(data), err + }, + // ReadLine doesn't fit the data/pattern easily + // so we leave it out. It should be covered via + // the ReadSlice test since ReadLine simply calls + // ReadSlice, and it's that function that handles + // the last byte. + } + + // Try all readers with UnreadByte. + for rno, read := range readers { + // Some input data that is longer than the minimum reader buffer size. + const n = 10 + var buf bytes.Buffer + for i := 0; i < n; i++ { + buf.WriteString("abcdefg") + } + + r := NewReaderSize(&buf, minReadBufferSize) + readTo := func(delim byte, want string) { + data, err := read(r, delim) + if err != nil { + t.Fatalf("#%d: unexpected error reading to %c: %v", rno, delim, err) + } + if got := string(data); got != want { + t.Fatalf("#%d: got %q, want %q", rno, got, want) + } + } + + // Read the data with occasional UnreadByte calls. + for i := 0; i < n; i++ { + readTo('d', "abcd") + for j := 0; j < 3; j++ { + if err := r.UnreadByte(); err != nil { + t.Fatalf("#%d: unexpected error on UnreadByte: %v", rno, err) + } + readTo('d', "d") + } + readTo('g', "efg") + } + + // All data should have been read. + _, err := r.ReadByte() + if err != io.EOF { + t.Errorf("#%d: got error %v; want EOF", rno, err) + } } } @@ -1056,7 +1169,61 @@ func TestWriterReadFromWhileFull(t *testing.T) { // Use ReadFrom to read in some data. n2, err := w.ReadFrom(strings.NewReader("abcdef")) if n2 != 6 || err != nil { - t.Fatalf("ReadFrom returned (%v, %v), want (6, nil)", n, err) + t.Fatalf("ReadFrom returned (%v, %v), want (6, nil)", n2, err) + } +} + +type emptyThenNonEmptyReader struct { + r io.Reader + n int +} + +func (r *emptyThenNonEmptyReader) Read(p []byte) (int, error) { + if r.n <= 0 { + return r.r.Read(p) + } + r.n-- + return 0, nil +} + +// Test for golang.org/issue/7611 +func TestWriterReadFromUntilEOF(t *testing.T) { + buf := new(bytes.Buffer) + w := NewWriterSize(buf, 5) + + // Partially fill buffer + n, err := w.Write([]byte("0123")) + if n != 4 || err != nil { + t.Fatalf("Write returned (%v, %v), want (4, nil)", n, err) + } + + // Use ReadFrom to read in some data. + r := &emptyThenNonEmptyReader{r: strings.NewReader("abcd"), n: 3} + n2, err := w.ReadFrom(r) + if n2 != 4 || err != nil { + t.Fatalf("ReadFrom returned (%v, %v), want (4, nil)", n2, err) + } + w.Flush() + if got, want := string(buf.Bytes()), "0123abcd"; got != want { + t.Fatalf("buf.Bytes() returned %q, want %q", got, want) + } +} + +func TestWriterReadFromErrNoProgress(t *testing.T) { + buf := new(bytes.Buffer) + w := NewWriterSize(buf, 5) + + // Partially fill buffer + n, err := w.Write([]byte("0123")) + if n != 4 || err != nil { + t.Fatalf("Write returned (%v, %v), want (4, nil)", n, err) + } + + // Use ReadFrom to read in some data. + r := &emptyThenNonEmptyReader{r: strings.NewReader("abcd"), n: 100} + n2, err := w.ReadFrom(r) + if n2 != 0 || err != io.ErrNoProgress { + t.Fatalf("buf.Bytes() returned (%v, %v), want (0, io.ErrNoProgress)", n2, err) } } @@ -1094,20 +1261,12 @@ func TestWriterReset(t *testing.T) { // An onlyReader only implements io.Reader, no matter what other methods the underlying implementation may have. type onlyReader struct { - r io.Reader -} - -func (r onlyReader) Read(b []byte) (int, error) { - return r.r.Read(b) + io.Reader } // An onlyWriter only implements io.Writer, no matter what other methods the underlying implementation may have. type onlyWriter struct { - w io.Writer -} - -func (w onlyWriter) Write(b []byte) (int, error) { - return w.w.Write(b) + io.Writer } func BenchmarkReaderCopyOptimal(b *testing.B) { @@ -1152,6 +1311,27 @@ func BenchmarkReaderCopyNoWriteTo(b *testing.B) { } } +func BenchmarkReaderWriteToOptimal(b *testing.B) { + const bufSize = 16 << 10 + buf := make([]byte, bufSize) + r := bytes.NewReader(buf) + srcReader := NewReaderSize(onlyReader{r}, 1<<10) + if _, ok := ioutil.Discard.(io.ReaderFrom); !ok { + b.Fatal("ioutil.Discard doesn't support ReaderFrom") + } + for i := 0; i < b.N; i++ { + r.Seek(0, 0) + srcReader.Reset(onlyReader{r}) + n, err := srcReader.WriteTo(ioutil.Discard) + if err != nil { + b.Fatal(err) + } + if n != bufSize { + b.Fatalf("n = %d; want %d", n, bufSize) + } + } +} + func BenchmarkWriterCopyOptimal(b *testing.B) { // Optimal case is where the underlying writer implements io.ReaderFrom srcBuf := bytes.NewBuffer(make([]byte, 8192))

b/libgo/go/bufio/scan.go diff --git a/libgo/go/bufio/scan.go (file)

index 77b2c2a .. 715ce07 100644 --- a/ libgo/go/bufio/scan.go +++ b/ libgo/go/bufio/scan.go @@ -135,7 +135,7 @@ func (s *Scanner) Scan() bool { } // Must read more data. // First, shift data to beginning of buffer if there's lots of empty space - // or space is neded. + // or space is ne e ded. if s.start > 0 && (s.end == len(s.buf) || s.start > len(s.buf)/2) { copy(s.buf, s.buf[s.start:s.end]) s.end -= s.start @@ -172,7 +172,7 @@ func (s *Scanner) Scan() bool { break } loop++ - if loop > 100 { + if loop > maxConsecutiveEmptyReads { s.setErr(io.ErrNoProgress) break }

b/libgo/go/bufio/scan_test.go diff --git a/libgo/go/bufio/scan_test.go (file)

index 4ac529f .. 0db7cad 100644 --- a/ libgo/go/bufio/scan_test.go +++ b/ libgo/go/bufio/scan_test.go @@ -277,7 +277,7 @@ func TestScanLineNoNewline(t *testing.T) { testNoNewline(text, lines, t) } -// Test that the line splitter handles a final line with a carriage return but nonewline. +// Test that the line splitter handles a final line with a carriage return but no newline. func TestScanLineReturnButNoNewline(t *testing.T) { const text = "abcdefghijklmn

opqrstuvwxyz\r" lines := []string{

b/libgo/go/bytes/bytes.go diff --git a/libgo/go/bytes/bytes.go (file)

index 644bf75 .. 0c53e4c 100644 --- a/ libgo/go/bytes/bytes.go +++ b/ libgo/go/bytes/bytes.go @@ -356,7 +356,11 @@ func Map(mapping func(r rune) rune, s []byte) []byte { } r = mapping(r) if r >= 0 { - if nbytes+utf8.RuneLen(r) > maxbytes { + rl := utf8.RuneLen(r) + if rl < 0 { + rl = len(string(utf8.RuneError)) + } + if nbytes+rl > maxbytes { // Grow the buffer. maxbytes = maxbytes*2 + utf8.UTFMax nb := make([]byte, maxbytes)

b/libgo/go/bytes/bytes_test.go diff --git a/libgo/go/bytes/bytes_test.go (file)

index 808655a .. 394dd7a 100644 --- a/ libgo/go/bytes/bytes_test.go +++ b/ libgo/go/bytes/bytes_test.go @@ -785,6 +785,16 @@ func TestMap(t *testing.T) { if string(m) != expect { t.Errorf("drop: expected %q got %q", expect, m) } + + // 6. Invalid rune + invalidRune := func(r rune) rune { + return utf8.MaxRune + 1 + } + m = Map(invalidRune, []byte("x")) + expect = "\uFFFD" + if string(m) != expect { + t.Errorf("invalidRune: expected %q got %q", expect, m) + } } func TestToUpper(t *testing.T) { runStringTests(t, ToUpper, "ToUpper", upperTests) } @@ -1134,7 +1144,7 @@ func TestEqualFold(t *testing.T) { func TestBufferGrowNegative(t *testing.T) { defer func() { if err := recover(); err == nil { - t.Fatal("Grow(-1) should have paniced") + t.Fatal("Grow(-1) should have panic k ed") } }() var b Buffer @@ -1144,7 +1154,7 @@ func TestBufferGrowNegative(t *testing.T) { func TestBufferTruncateNegative(t *testing.T) { defer func() { if err := recover(); err == nil { - t.Fatal("Truncate(-1) should have paniced") + t.Fatal("Truncate(-1) should have panic k ed") } }() var b Buffer @@ -1154,7 +1164,7 @@ func TestBufferTruncateNegative(t *testing.T) { func TestBufferTruncateOutOfRange(t *testing.T) { defer func() { if err := recover(); err == nil { - t.Fatal("Truncate(20) should have paniced") + t.Fatal("Truncate(20) should have panic k ed") } }() var b Buffer

b/libgo/go/bytes/reader.go diff --git a/libgo/go/bytes/reader.go (file)

index 77511b9 .. d2d40fa 100644 --- a/ libgo/go/bytes/reader.go +++ b/ libgo/go/bytes/reader.go @@ -16,40 +16,41 @@ import ( // Unlike a Buffer, a Reader is read-only and supports seeking. type Reader struct { s []byte - i int // current reading index - prevRune int // index of previous rune; or < 0 + i int 64 // current reading index + prevRune int // index of previous rune; or < 0 } // Len returns the number of bytes of the unread portion of the // slice. func (r *Reader) Len() int { - if r.i >= len(r.s ) { + if r.i >= int64(len(r.s) ) { return 0 } - return len(r.s) - r.i + return int(int64(len(r.s)) - r.i) } func (r *Reader) Read(b []byte) (n int, err error) { if len(b) == 0 { return 0, nil } - if r.i >= len(r.s ) { + if r.i >= int64(len(r.s) ) { return 0, io.EOF } - n = copy(b, r.s[r.i:]) - r.i += n r.prevRune = -1 + n = copy(b, r.s[r.i:]) + r.i += int64(n) return } func (r *Reader) ReadAt(b []byte, off int64) (n int, err error) { + // cannot modify state - see io.ReaderAt if off < 0 { - return 0, errors.New("bytes : invalid offset") + return 0, errors.New("bytes .Reader.ReadAt: negative offset") } if off >= int64(len(r.s)) { return 0, io.EOF } - n = copy(b, r.s[ int(off) :]) + n = copy(b, r.s[ off :]) if n < len(b) { err = io.EOF } @@ -57,49 +58,51 @@ func (r *Reader) ReadAt(b []byte, off int64) (n int, err error) { } func (r *Reader) ReadByte() (b byte, err error) { - if r.i >= len(r.s) { + r.prevRune = -1 + if r.i >= int64(len(r.s)) { return 0, io.EOF } b = r.s[r.i] r.i++ - r.prevRune = -1 return } func (r *Reader) UnreadByte() error { + r.prevRune = -1 if r.i <= 0 { - return errors.New("bytes.Reader: at beginning of slice") + return errors.New("bytes.Reader .UnreadByte : at beginning of slice") } r.i-- - r.prevRune = -1 return nil } func (r *Reader) ReadRune() (ch rune, size int, err error) { - if r.i >= len(r.s) { + if r.i >= int64(len(r.s)) { + r.prevRune = -1 return 0, 0, io.EOF } - r.prevRune = r.i + r.prevRune = int(r.i) if c := r.s[r.i]; c < utf8.RuneSelf { r.i++ return rune(c), 1, nil } ch, size = utf8.DecodeRune(r.s[r.i:]) - r.i += size + r.i += int64(size) return } func (r *Reader) UnreadRune() error { if r.prevRune < 0 { - return errors.New("bytes.Reader: previous operation was not ReadRune") + return errors.New("bytes.Reader .UnreadRune : previous operation was not ReadRune") } - r.i = r.prevRune + r.i = int64(r.prevRune) r.prevRune = -1 return nil } // Seek implements the io.Seeker interface. func (r *Reader) Seek(offset int64, whence int) (int64, error) { + r.prevRune = -1 var abs int64 switch whence { case 0: @@ -109,22 +112,19 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) { case 2: abs = int64(len(r.s)) + offset default: - return 0, errors.New("bytes: invalid whence") + return 0, errors.New("bytes .Reader.Seek : invalid whence") } if abs < 0 { - return 0, errors.New("bytes: negative position") - } - if abs >= 1<<31 { - return 0, errors.New("bytes: position out of range") + return 0, errors.New("bytes.Reader.Seek: negative position") } - r.i = int(abs) + r.i = abs return abs, nil } // WriteTo implements the io.WriterTo interface. func (r *Reader) WriteTo(w io.Writer) (n int64, err error) { r.prevRune = -1 - if r.i >= len(r.s ) { + if r.i >= int64(len(r.s) ) { return 0, nil } b := r.s[r.i:] @@ -132,7 +132,7 @@ func (r *Reader) WriteTo(w io.Writer) (n int64, err error) { if m > len(b) { panic("bytes.Reader.WriteTo: invalid Write count") } - r.i += m + r.i += int64(m) n = int64(m) if m != len(b) && err == nil { err = io.ErrShortWrite

b/libgo/go/bytes/reader_test.go diff --git a/libgo/go/bytes/reader_test.go (file)

index 19f014d .. d3dce53 100644 --- a/ libgo/go/bytes/reader_test.go +++ b/ libgo/go/bytes/reader_test.go @@ -10,6 +10,7 @@ import ( "io" "io/ioutil" "os" + "sync" "testing" ) @@ -26,9 +27,9 @@ func TestReader(t *testing.T) { {seek: os.SEEK_SET, off: 0, n: 20, want: "0123456789"}, {seek: os.SEEK_SET, off: 1, n: 1, want: "1"}, {seek: os.SEEK_CUR, off: 1, wantpos: 3, n: 2, want: "34"}, - {seek: os.SEEK_SET, off: -1, seekerr: "bytes: negative position"}, - {seek: os.SEEK_SET, off: 1 <<31 - 1 }, - {seek: os.SEEK_CUR, off: 1, seekerr: "bytes: position out of range" }, + {seek: os.SEEK_SET, off: -1, seekerr: "bytes .Reader.Seek : negative position"}, + {seek: os.SEEK_SET, off: 1 << 33, wantpos: 1 << 33 }, + {seek: os.SEEK_CUR, off: 1, wantpos: 1<<33 + 1 }, {seek: os.SEEK_SET, n: 5, want: "01234"}, {seek: os.SEEK_CUR, n: 5, want: "56789"}, {seek: os.SEEK_END, off: -1, n: 1, wantpos: 9, want: "9"}, @@ -60,6 +61,16 @@ func TestReader(t *testing.T) { } } +func TestReadAfterBigSeek(t *testing.T) { + r := NewReader([]byte("0123456789")) + if _, err := r.Seek(1<<31+5, os.SEEK_SET); err != nil { + t.Fatal(err) + } + if n, err := r.Read(make([]byte, 10)); n != 0 || err != io.EOF { + t.Errorf("Read = %d, %v; want 0, EOF", n, err) + } +} + func TestReaderAt(t *testing.T) { r := NewReader([]byte("0123456789")) tests := []struct { @@ -73,7 +84,7 @@ func TestReaderAt(t *testing.T) { {1, 9, "123456789", nil}, {11, 10, "", io.EOF}, {0, 0, "", nil}, - {-1, 0, "", "bytes : invalid offset"}, + {-1, 0, "", "bytes .Reader.ReadAt: negative offset"}, } for i, tt := range tests { b := make([]byte, tt.n) @@ -88,6 +99,43 @@ func TestReaderAt(t *testing.T) { } } +func TestReaderAtConcurrent(t *testing.T) { + // Test for the race detector, to verify ReadAt doesn't mutate + // any state. + r := NewReader([]byte("0123456789")) + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + var buf [1]byte + r.ReadAt(buf[:], int64(i)) + }(i) + } + wg.Wait() +} + +func TestEmptyReaderConcurrent(t *testing.T) { + // Test for the race detector, to verify a Read that doesn't yield any bytes + // is okay to use from multiple goroutines. This was our historic behavior. + // See golang.org/issue/7856 + r := NewReader([]byte{}) + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(2) + go func() { + defer wg.Done() + var buf [1]byte + r.Read(buf[:]) + }() + go func() { + defer wg.Done() + r.Read(nil) + }() + } + wg.Wait() +} + func TestReaderWriteTo(t *testing.T) { for i := 0; i < 30; i += 3 { var l int @@ -133,6 +181,32 @@ func TestReaderLen(t *testing.T) { } } +var UnreadRuneErrorTests = []struct { + name string + f func(*Reader) +}{ + {"Read", func(r *Reader) { r.Read([]byte{0}) }}, + {"ReadByte", func(r *Reader) { r.ReadByte() }}, + {"UnreadRune", func(r *Reader) { r.UnreadRune() }}, + {"Seek", func(r *Reader) { r.Seek(0, 1) }}, + {"WriteTo", func(r *Reader) { r.WriteTo(&Buffer{}) }}, +} + +func TestUnreadRuneError(t *testing.T) { + for _, tt := range UnreadRuneErrorTests { + reader := NewReader([]byte("0123456789")) + if _, _, err := reader.ReadRune(); err != nil { + // should not happen + t.Fatal(err) + } + tt.f(reader) + err := reader.UnreadRune() + if err == nil { + t.Errorf("Unreading after %s: expected error", tt.name) + } + } +} + func TestReaderDoubleUnreadRune(t *testing.T) { buf := NewBuffer([]byte("groucho")) if _, _, err := buf.ReadRune(); err != nil {

b/libgo/go/compress/bzip2/bzip2_test.go diff --git a/libgo/go/compress/bzip2/bzip2_test.go (file)

index cd647e5 .. 727249d 100644 --- a/ libgo/go/compress/bzip2/bzip2_test.go +++ b/ libgo/go/compress/bzip2/bzip2_test.go @@ -177,7 +177,7 @@ const ( var testfiles = []string{ // Digits is the digits of the irrational number e. Its decimal representation - // does not repeat, but there are only 10 posible digits, so it should be + // does not repeat, but there are only 10 pos s ible digits, so it should be // reasonably compressible. digits: "testdata/e.txt.bz2", // Twain is Project Gutenberg's edition of Mark Twain's classic English novel.

b/libgo/go/compress/flate/inflate.go diff --git a/libgo/go/compress/flate/inflate.go (file)

index bbe4c5a .. ce4923e 100644 --- a/ libgo/go/compress/flate/inflate.go +++ b/ libgo/go/compress/flate/inflate.go @@ -54,7 +54,7 @@ func (e *WriteError) Error() string { return "flate: write error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error() } -// Note that much of the implemenation of huffmanDecoder is also copied +// Note that much of the implemen t ation of huffmanDecoder is also copied // into gen.go (in package main) for the purpose of precomputing the // fixed huffman tables so they can be included statically.

b/libgo/go/compress/flate/reader_test.go diff --git a/libgo/go/compress/flate/reader_test.go (file)

index 2a8ebbc .. a62ef74 100644 --- a/ libgo/go/compress/flate/reader_test.go +++ b/ libgo/go/compress/flate/reader_test.go @@ -29,7 +29,7 @@ const ( var testfiles = []string{ // Digits is the digits of the irrational number e. Its decimal representation - // does not repeat, but there are only 10 posible digits, so it should be + // does not repeat, but there are only 10 pos s ible digits, so it should be // reasonably compressible. digits: "../testdata/e.txt", // Twain is Project Gutenberg's edition of Mark Twain's classic English novel.

b/libgo/go/compress/gzip/gunzip.go diff --git a/libgo/go/compress/gzip/gunzip.go (file)

index 1fb9b09 .. 4f398b1 100644 --- a/ libgo/go/compress/gzip/gunzip.go +++ b/ libgo/go/compress/gzip/gunzip.go @@ -89,6 +89,21 @@ func NewReader(r io.Reader) (*Reader, error) { return z, nil } +// Reset discards the Reader z's state and makes it equivalent to the +// result of its original state from NewReader, but reading from r instead. +// This permits reusing a Reader rather than allocating a new one. +func (z *Reader) Reset(r io.Reader) error { + z.r = makeReader(r) + if z.digest == nil { + z.digest = crc32.NewIEEE() + } else { + z.digest.Reset() + } + z.size = 0 + z.err = nil + return z.readHeader(true) +} + // GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950). func get4(p []byte) uint32 { return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24

b/libgo/go/compress/gzip/gunzip_test.go diff --git a/libgo/go/compress/gzip/gunzip_test.go (file)

index 5615373 .. 2471038 100644 --- a/ libgo/go/compress/gzip/gunzip_test.go +++ b/ libgo/go/compress/gzip/gunzip_test.go @@ -303,6 +303,26 @@ func TestDecompressor(t *testing.T) { if s != tt.raw { t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.name, n, s, len(tt.raw), tt.raw) } + + // Test Reader Reset. + in = bytes.NewReader(tt.gzip) + err = gzip.Reset(in) + if err != nil { + t.Errorf("%s: Reset: %s", tt.name, err) + continue + } + if tt.name != gzip.Name { + t.Errorf("%s: got name %s", tt.name, gzip.Name) + } + b.Reset() + n, err = io.Copy(b, gzip) + if err != tt.err { + t.Errorf("%s: io.Copy: %v want %v", tt.name, err, tt.err) + } + s = b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.name, n, s, len(tt.raw), tt.raw) + } } } @@ -333,3 +353,17 @@ func TestIssue6550(t *testing.T) { // ok } } + +func TestInitialReset(t *testing.T) { + var r Reader + if err := r.Reset(bytes.NewReader(gunzipTests[1].gzip)); err != nil { + t.Error(err) + } + var buf bytes.Buffer + if _, err := io.Copy(&buf, &r); err != nil { + t.Error(err) + } + if s := buf.String(); s != gunzipTests[1].raw { + t.Errorf("got %q want %q", s, gunzipTests[1].raw) + } +}

b/libgo/go/compress/gzip/gzip.go diff --git a/libgo/go/compress/gzip/gzip.go (file)

index fe32d68 .. 3a0bf54 100644 --- a/ libgo/go/compress/gzip/gzip.go +++ b/ libgo/go/compress/gzip/gzip.go @@ -22,8 +22,8 @@ const ( DefaultCompression = flate.DefaultCompression ) -// A Writer is an io.WriteCloser that satisfies writes by compressing data written -// to its wrapped io.Writer . +// A Writer is an io.WriteCloser . +// Writes to a Writer are compressed and written to w . type Writer struct { Header w io.Writer @@ -37,8 +37,8 @@ type Writer struct { err error } -// NewWriter creates a new Writer that satisfies writes by compressing data -// written to w. +// NewWriter returns a new Writer. +// Writes to the returned writer are compressed and written to w. // // It is the caller's responsibility to call Close on the WriteCloser when done. // Writes may be buffered and not flushed until Close.

b/libgo/go/compress/lzw/reader.go diff --git a/libgo/go/compress/lzw/reader.go (file)

index efbc758 .. ef59699 100644 --- a/ libgo/go/compress/lzw/reader.go +++ b/ libgo/go/compress/lzw/reader.go @@ -216,8 +216,8 @@ func (d *decoder) Close() error { return nil } -// NewReader creates a new io.ReadCloser that satisfies reads by decompressing -// the data read from r. +// NewReader creates a new io.ReadCloser . +// Reads from the returned io.ReadCloser read and decompress data from r. // It is the caller's responsibility to call Close on the ReadCloser when // finished reading. // The number of bits to use for literal codes, litWidth, must be in the

b/libgo/go/compress/lzw/writer.go diff --git a/libgo/go/compress/lzw/writer.go (file)

index b206918 .. 961b25f 100644 --- a/ libgo/go/compress/lzw/writer.go +++ b/ libgo/go/compress/lzw/writer.go @@ -225,8 +225,8 @@ func (e *encoder) Close() error { return e.w.Flush() } -// NewWriter creates a new io.WriteCloser that satisfies writes by compressing -// the data and writing it to w. +// NewWriter creates a new io.WriteCloser . +// Writes to the returned io.WriteCloser are compressed and written to w. // It is the caller's responsibility to call Close on the WriteCloser when // finished writing. // The number of bits to use for literal codes, litWidth, must be in the

b/libgo/go/compress/zlib/reader.go diff --git a/libgo/go/compress/zlib/reader.go (file)

index d54746f .. 9e1aafd 100644 --- a/ libgo/go/compress/zlib/reader.go +++ b/ libgo/go/compress/zlib/reader.go @@ -51,7 +51,8 @@ type reader struct { scratch [4]byte } -// NewReader creates a new io.ReadCloser that satisfies reads by decompressing data read from r. +// NewReader creates a new io.ReadCloser. +// Reads from the returned io.ReadCloser read and decompress data from r. // The implementation buffers input and may read more data than necessary from r. // It is the caller's responsibility to call Close on the ReadCloser when done. func NewReader(r io.Reader) (io.ReadCloser, error) {

b/libgo/go/compress/zlib/writer.go diff --git a/libgo/go/compress/zlib/writer.go (file)

index 99ff654 .. fac7e15 100644 --- a/ libgo/go/compress/zlib/writer.go +++ b/ libgo/go/compress/zlib/writer.go @@ -34,8 +34,8 @@ type Writer struct { wroteHeader bool } -// NewWriter creates a new Writer that satisfies writes by compressing data -// written to w. +// NewWriter creates a new Writer . +// Writes to the returned Writer are compressed and written to w. // // It is the caller's responsibility to call Close on the WriteCloser when done. // Writes may be buffered and not flushed until Close.

b/libgo/go/compress/zlib/writer_test.go diff --git a/libgo/go/compress/zlib/writer_test.go (file)

index cf9c832 .. 71ba81a 100644 --- a/ libgo/go/compress/zlib/writer_test.go +++ b/ libgo/go/compress/zlib/writer_test.go @@ -120,7 +120,7 @@ func testFileLevelDictReset(t *testing.T, fn string, level int, dict []byte) { } out := buf.String() - // Reset and compr se s again. + // Reset and compr es s again. buf2 := new(bytes.Buffer) zlibw.Reset(buf2) _, err = zlibw.Write(b0)

b/libgo/go/container/heap/heap.go diff --git a/libgo/go/container/heap/heap.go (file)

index 3fe2327 .. c467a11 100644 --- a/ libgo/go/container/heap/heap.go +++ b/ libgo/go/container/heap/heap.go @@ -78,7 +78,7 @@ func Remove(h Interface, i int) interface{} { return h.Pop() } -// Fix reestablishes the heap ordering after the element at index i has changed its value. +// Fix re - establishes the heap ordering after the element at index i has changed its value. // Changing the value of the element at index i and then calling Fix is equivalent to, // but less expensive than, calling Remove(h, i) followed by a Push of the new value. // The complexity is O(log(n)) where n = h.Len().

b/libgo/go/crypto/aes/aes_test.go diff --git a/libgo/go/crypto/aes/aes_test.go (file)

index 6261dd0 .. 3631809 100644 --- a/ libgo/go/crypto/aes/aes_test.go +++ b/ libgo/go/crypto/aes/aes_test.go @@ -354,6 +354,34 @@ func TestCipherDecrypt(t *testing.T) { } } +// Test short input/output. +// Assembly used to not notice. +// See issue 7928. +func TestShortBlocks(t *testing.T) { + bytes := func(n int) []byte { return make([]byte, n) } + + c, _ := NewCipher(bytes(16)) + + mustPanic(t, "crypto/aes: input not full block", func() { c.Encrypt(bytes(1), bytes(1)) }) + mustPanic(t, "crypto/aes: input not full block", func() { c.Decrypt(bytes(1), bytes(1)) }) + mustPanic(t, "crypto/aes: input not full block", func() { c.Encrypt(bytes(100), bytes(1)) }) + mustPanic(t, "crypto/aes: input not full block", func() { c.Decrypt(bytes(100), bytes(1)) }) + mustPanic(t, "crypto/aes: output not full block", func() { c.Encrypt(bytes(1), bytes(100)) }) + mustPanic(t, "crypto/aes: output not full block", func() { c.Decrypt(bytes(1), bytes(100)) }) +} + +func mustPanic(t *testing.T, msg string, f func()) { + defer func() { + err := recover() + if err == nil { + t.Errorf("function did not panic, wanted %q", msg) + } else if err != msg { + t.Errorf("got panic %v, wanted %q", err, msg) + } + }() + f() +} + func BenchmarkEncrypt(b *testing.B) { tt := encryptTests[0] c, err := NewCipher(tt.key)

b/libgo/go/crypto/aes/cipher.go diff --git a/libgo/go/crypto/aes/cipher.go (file)

index d931134 .. 2c6bb0a 100644 --- a/ libgo/go/crypto/aes/cipher.go +++ b/ libgo/go/crypto/aes/cipher.go @@ -46,9 +46,21 @@ func NewCipher(key []byte) (cipher.Block, error) { func (c *aesCipher) BlockSize() int { return BlockSize } func (c *aesCipher) Encrypt(dst, src []byte) { + if len(src) < BlockSize { + panic("crypto/aes: input not full block") + } + if len(dst) < BlockSize { + panic("crypto/aes: output not full block") + } encryptBlock(c.enc, dst, src) } func (c *aesCipher) Decrypt(dst, src []byte) { + if len(src) < BlockSize { + panic("crypto/aes: input not full block") + } + if len(dst) < BlockSize { + panic("crypto/aes: output not full block") + } decryptBlock(c.dec, dst, src) }

b/libgo/go/crypto/cipher/benchmark_test.go diff --git a/libgo/go/crypto/cipher/benchmark_test.go (file)

index 0b173a4 .. 027b248 100644 --- a/ libgo/go/crypto/cipher/benchmark_test.go +++ b/ libgo/go/crypto/cipher/benchmark_test.go @@ -47,7 +47,7 @@ func BenchmarkAESGCMOpen1K(b *testing.B) { } // If we test exactly 1K blocks, we would generate exact multiples of -// the cipher's block size, and and the cipher stream fragments would +// the cipher's block size, and the cipher stream fragments would // always be wordsize aligned, whereas non-aligned is a more typical // use-case. const almost1K = 1024 - 5

b/libgo/go/crypto/cipher/gcm.go diff --git a/libgo/go/crypto/cipher/gcm.go (file)

index 2f748f0 .. bdafd85 100644 --- a/ libgo/go/crypto/cipher/gcm.go +++ b/ libgo/go/crypto/cipher/gcm.go @@ -30,9 +30,9 @@ type AEAD interface { // Open decrypts and authenticates ciphertext, authenticates the // additional data and, if successful, appends the resulting plaintext - // to dst, returning the updated slice and true. On error, nil and - // false is returned. The nonce must be NonceSize() bytes long and both - // it and the additional data must match the value passed to Seal. + // to dst, returning the updated slice . The nonce must be NonceSize() + // bytes long and both it and the additional data must match the + // value passed to Seal. // // The ciphertext and dst may alias exactly or not at all. Open(dst, nonce, ciphertext, data []byte) ([]byte, error)

b/libgo/go/crypto/dsa/dsa.go diff --git a/libgo/go/crypto/dsa/dsa.go (file)

index 5a2a657 .. b7565a6 100644 --- a/ libgo/go/crypto/dsa/dsa.go +++ b/ libgo/go/crypto/dsa/dsa.go @@ -173,6 +173,16 @@ func GenerateKey(priv *PrivateKey, rand io.Reader) error { return nil } +// fermatInverse calculates the inverse of k in GF(P) using Fermat's method. +// This has better constant-time properties than Euclid's method (implemented +// in math/big.Int.ModInverse) although math/big itself isn't strictly +// constant-time so it's not perfect. +func fermatInverse(k, P *big.Int) *big.Int { + two := big.NewInt(2) + pMinus2 := new(big.Int).Sub(P, two) + return new(big.Int).Exp(k, pMinus2, P) +} + // Sign signs an arbitrary length hash (which should be the result of hashing a // larger message) using the private key, priv. It returns the signature as a // pair of integers. The security of the private key depends on the entropy of @@ -205,7 +215,7 @@ func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err err } } - kInv := new(big.Int).Mod Inverse(k, priv.Q) + kInv := fermat Inverse(k, priv.Q) r = new(big.Int).Exp(priv.G, k, priv.P) r.Mod(r, priv.Q)

b/libgo/go/crypto/ecdsa/ecdsa.go diff --git a/libgo/go/crypto/ecdsa/ecdsa.go (file)

index d02f15c .. 1bec743 100644 --- a/ libgo/go/crypto/ecdsa/ecdsa.go +++ b/ libgo/go/crypto/ecdsa/ecdsa.go @@ -84,6 +84,16 @@ func hashToInt(hash []byte, c elliptic.Curve) *big.Int { return ret } +// fermatInverse calculates the inverse of k in GF(P) using Fermat's method. +// This has better constant-time properties than Euclid's method (implemented +// in math/big.Int.ModInverse) although math/big itself isn't strictly +// constant-time so it's not perfect. +func fermatInverse(k, N *big.Int) *big.Int { + two := big.NewInt(2) + nMinus2 := new(big.Int).Sub(N, two) + return new(big.Int).Exp(k, nMinus2, N) +} + // Sign signs an arbitrary length hash (which should be the result of hashing a // larger message) using the private key, priv. It returns the signature as a // pair of integers. The security of the private key depends on the entropy of @@ -102,7 +112,7 @@ func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err err return } - kInv = new(big.Int).Mod Inverse(k, N) + kInv = fermat Inverse(k, N) r, _ = priv.Curve.ScalarBaseMult(k.Bytes()) r.Mod(r, N) if r.Sign() != 0 {

b/libgo/go/crypto/md5/md5block_decl.go diff --git a/libgo/go/crypto/md5/md5block_decl.go (file)

index c4d6aaa .. d7956a6 100644 --- a/ libgo/go/crypto/md5/md5block_decl.go +++ b/ libgo/go/crypto/md5/md5block_decl.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build amd64 386 arm +// +build amd64 amd64p32 386 arm package md5

b/libgo/go/crypto/md5/md5block_generic.go diff --git a/libgo/go/crypto/md5/md5block_generic.go (file)

index 239bf4d .. 263463e 100644 --- a/ libgo/go/crypto/md5/md5block_generic.go +++ b/ libgo/go/crypto/md5/md5block_generic.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !amd64,!386,!arm +// +build !amd64,! amd64p32,! 386,!arm package md5

b/libgo/go/crypto/rand/rand_unix.go diff --git a/libgo/go/crypto/rand/rand_unix.go (file)

index 0fbd7ea .. 1e741fd 100644 --- a/ libgo/go/crypto/rand/rand_unix.go +++ b/ libgo/go/crypto/rand/rand_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd plan9 solaris +// +build darwin dragonfly freebsd linux n acl n etbsd openbsd plan9 solaris // Unix cryptographically secure pseudorandom number // generator.

b/libgo/go/crypto/rc4/rc4_asm.go diff --git a/libgo/go/crypto/rc4/rc4_asm.go (file)

index c582a44 .. fc71b9a 100644 --- a/ libgo/go/crypto/rc4/rc4_asm.go +++ b/ libgo/go/crypto/rc4/rc4_asm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build amd64 arm 386 +// +build amd64 a md64p32 a rm 386 package rc4

b/libgo/go/crypto/rc4/rc4_ref.go diff --git a/libgo/go/crypto/rc4/rc4_ref.go (file)

index bdf5e1d .. 1ecce1a 100644 --- a/ libgo/go/crypto/rc4/rc4_ref.go +++ b/ libgo/go/crypto/rc4/rc4_ref.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !amd64,!arm,!386 +// +build !amd64,!a md64p32,!a rm,!386 package rc4

b/libgo/go/crypto/rsa/pkcs1v15.go diff --git a/libgo/go/crypto/rsa/pkcs1v15.go (file)

index cf174b6 .. d9957ae 100644 --- a/ libgo/go/crypto/rsa/pkcs1v15.go +++ b/ libgo/go/crypto/rsa/pkcs1v15.go @@ -214,7 +214,7 @@ func SignPKCS1v15(rand io.Reader, priv *PrivateKey, hash crypto.Hash, hashed []b // hashed is the result of hashing the input message using the given hash // function and sig is the signature. A valid signature is indicated by // returning a nil error. If hash is zero then hashed is used directly. This -// isn't advisable except for interopability. +// isn't advisable except for interop er ability. func VerifyPKCS1v15(pub *PublicKey, hash crypto.Hash, hashed []byte, sig []byte) (err error) { hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed)) if err != nil {

b/libgo/go/crypto/rsa/pss.go diff --git a/libgo/go/crypto/rsa/pss.go (file)

index f9abec3 .. 18eafbc 100644 --- a/ libgo/go/crypto/rsa/pss.go +++ b/ libgo/go/crypto/rsa/pss.go @@ -4,7 +4,7 @@ package rsa -// This file implement e s the PSS signature scheme [1]. +// This file implements the PSS signature scheme [1]. // // [1] http://www.rsa.com/rsalabs/pkcs/files/h11300-wp-pkcs-1v2-2-rsa-cryptography-standard.pdf @@ -189,7 +189,7 @@ func emsaPSSVerify(mHash, em []byte, emBits, sLen int, hash hash.Hash) error { // signPSSWithSalt calculates the signature of hashed using PSS [1] with specified salt. // Note that hashed must be the result of hashing the input message using the -// given hash funcion. salt is a random sequence of bytes whose length will be +// given hash func t ion. salt is a random sequence of bytes whose length will be // later used to verify the signature. func signPSSWithSalt(rand io.Reader, priv *PrivateKey, hash crypto.Hash, hashed, salt []byte) (s []byte, err error) { nBits := priv.N.BitLen() @@ -233,7 +233,7 @@ func (opts *PSSOptions) saltLength() int { // SignPSS calculates the signature of hashed using RSASSA-PSS [1]. // Note that hashed must be the result of hashing the input message using the -// given hash funcion. The opts argument may be nil, in which case sensible +// given hash func t ion. The opts argument may be nil, in which case sensible // defaults are used. func SignPSS(rand io.Reader, priv *PrivateKey, hash crypto.Hash, hashed []byte, opts *PSSOptions) (s []byte, err error) { saltLength := opts.saltLength()

b/libgo/go/crypto/rsa/rsa.go diff --git a/libgo/go/crypto/rsa/rsa.go (file)

index c8f1feb .. bce6ba4 100644 --- a/ libgo/go/crypto/rsa/rsa.go +++ b/ libgo/go/crypto/rsa/rsa.go @@ -60,7 +60,7 @@ type PrivateKey struct { type PrecomputedValues struct { Dp, Dq *big.Int // D mod (P-1) (or mod Q-1) - Qinv *big.Int // Q^-1 mod Q + Qinv *big.Int // Q^-1 mod P // CRTValues is used for the 3rd and subsequent primes. Due to a // historical accident, the CRT for the first two primes is handled

b/libgo/go/crypto/sha1/sha1block_decl.go diff --git a/libgo/go/crypto/sha1/sha1block_decl.go (file)

index b2c68f0 .. 24e521a 100644 --- a/ libgo/go/crypto/sha1/sha1block_decl.go +++ b/ libgo/go/crypto/sha1/sha1block_decl.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build amd64 386 arm +// +build amd64 amd64p32 arm 386 package sha1

b/libgo/go/crypto/sha1/sha1block_generic.go diff --git a/libgo/go/crypto/sha1/sha1block_generic.go (file)

index 2c78683 .. 696e26b 100644 --- a/ libgo/go/crypto/sha1/sha1block_generic.go +++ b/ libgo/go/crypto/sha1/sha1block_generic.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !amd64,!386,!arm +// +build !amd64,! amd64p32,! 386,!arm package sha1

b/libgo/go/crypto/tls/common.go diff --git a/libgo/go/crypto/tls/common.go (file)

index 7ce2077 .. fca98bd 100644 --- a/ libgo/go/crypto/tls/common.go +++ b/ libgo/go/crypto/tls/common.go @@ -82,12 +82,14 @@ const ( scsvRenegotiation uint16 = 0x00ff ) -// TLS Elliptic Curves +// CurveID is the type of a TLS identifier for an elliptic curve. See // http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-8 +type CurveID uint16 + const ( - curveP256 uint16 = 23 - curveP384 uint16 = 24 - curveP521 uint16 = 25 + CurveP256 CurveID = 23 + CurveP384 CurveID = 24 + CurveP521 CurveID = 25 ) // TLS Elliptic Curve Point Formats @@ -153,6 +155,7 @@ var supportedClientCertSignatureAlgorithms = []signatureAndHash{ // ConnectionState records basic TLS details about the connection. type ConnectionState struct { + Version uint16 // TLS version used by the connection (e.g. VersionTLS12) HandshakeComplete bool // TLS handshake is complete DidResume bool // connection resumes a previous TLS connection CipherSuite uint16 // cipher suite in use (TLS_RSA_WITH_RC4_128_SHA, ...) @@ -198,12 +201,15 @@ type ClientSessionCache interface { Put(sessionKey string, cs *ClientSessionState) } -// A Config structure is used to configure a TLS client or server. After one -// has been passed to a TLS function it must not be modified. +// A Config structure is used to configure a TLS client or server. +// After one has been passed to a TLS function it must not be +// modified. A Config may be reused; the tls package will also not +// modify it. type Config struct { // Rand provides the source of entropy for nonces and RSA blinding. // If Rand is nil, TLS uses the cryptographic random reader in package // crypto/rand. + // The Reader must be safe for use by multiple goroutines. Rand io.Reader // Time returns the current time as the number of seconds since the epoch. @@ -290,6 +296,11 @@ type Config struct { // which is currently TLS 1.2. MaxVersion uint16 + // CurvePreferences contains the elliptic curves that will be used in + // an ECDHE handshake, in preference order. If empty, the default will + // be used. + CurvePreferences []CurveID + serverInitOnce sync.Once // guards calling (*Config).serverInit } @@ -348,6 +359,15 @@ func (c *Config) maxVersion() uint16 { return c.MaxVersion } +var defaultCurvePreferences = []CurveID{CurveP256, CurveP384, CurveP521} + +func (c *Config) curvePreferences() []CurveID { + if c == nil || len(c.CurvePreferences) == 0 { + return defaultCurvePreferences + } + return c.CurvePreferences +} + // mutualVersion returns the protocol version to use given the advertised // version of the peer. func (c *Config) mutualVersion(vers uint16) (uint16, bool) {

b/libgo/go/crypto/tls/conn.go diff --git a/libgo/go/crypto/tls/conn.go (file)

index c33549c .. 8f7d2c1 100644 --- a/ libgo/go/crypto/tls/conn.go +++ b/ libgo/go/crypto/tls/conn.go @@ -28,6 +28,7 @@ type Conn struct { // constant after handshake; protected by handshakeMutex handshakeMutex sync.Mutex // handshakeMutex < in.Mutex, out.Mutex, errMutex + handshakeErr error // error resulting from handshake vers uint16 // TLS version haveVers bool // version has been negotiated config *Config // configuration passed to constructor @@ -45,9 +46,6 @@ type Conn struct { clientProtocol string clientProtocolFallback bool - // first permanent error - connErr - // input/output in, out halfConn // in.Mutex < out.Mutex rawInput *block // raw input, right off the wire @@ -57,27 +55,6 @@ type Conn struct { tmp [16]byte } -type connErr struct { - mu sync.Mutex - value error -} - -func (e *connErr) setError(err error) error { - e.mu.Lock() - defer e.mu.Unlock() - - if e.value == nil { - e.value = err - } - return err -} - -func (e *connErr) error() error { - e.mu.Lock() - defer e.mu.Unlock() - return e.value -} - // Access to net.Conn methods. // Cannot just embed net.Conn because that would // export the struct field too. @@ -105,7 +82,7 @@ func (c *Conn) SetReadDeadline(t time.Time) error { return c.conn.SetReadDeadline(t) } -// SetWriteDeadline sets the write deadline on the underlying connec it on. +// SetWriteDeadline sets the write deadline on the underlying connec ti on. // A zero value for t means Write will not time out. // After a Write has timed out, the TLS state is corrupt and all future writes will return the same error. func (c *Conn) SetWriteDeadline(t time.Time) error { @@ -116,6 +93,8 @@ func (c *Conn) SetWriteDeadline(t time.Time) error { // connection, either sending or receiving. type halfConn struct { sync.Mutex + + err error // first permanent error version uint16 // protocol version cipher interface{} // cipher algorithm mac macFunction @@ -129,6 +108,18 @@ type halfConn struct { inDigestBuf, outDigestBuf []byte } +func (hc *halfConn) setErrorLocked(err error) error { + hc.err = err + return err +} + +func (hc *halfConn) error() error { + hc.Lock() + err := hc.err + hc.Unlock() + return err +} + // prepareCipherSpec sets the encryption and MAC states // that a subsequent changeCipherSpec will use. func (hc *halfConn) prepareCipherSpec(version uint16, cipher interface{}, mac macFunction) { @@ -460,6 +451,8 @@ func (b *block) readFromUntil(r io.Reader, n int) error { m, err := r.Read(b.data[len(b.data):cap(b.data)]) b.data = b.data[0 : len(b.data)+m] if len(b.data) >= n { + // TODO(bradfitz,agl): slightly suspicious + // that we're throwing away r.Read's err here. break } if err != nil { @@ -520,16 +513,16 @@ func (c *Conn) readRecord(want recordType) error { switch want { default: c.sendAlert(alertInternalError) - return errors.New("tls: unknown record type requested" ) + return c.in.setErrorLocked(errors.New("tls: unknown record type requested") ) case recordTypeHandshake, recordTypeChangeCipherSpec: if c.handshakeComplete { c.sendAlert(alertInternalError) - return errors.New("tls: handshake or ChangeCipherSpec requested after handshake complete" ) + return c.in.setErrorLocked(errors.New("tls: handshake or ChangeCipherSpec requested after handshake complete") ) } case recordTypeApplicationData: if !c.handshakeComplete { c.sendAlert(alertInternalError) - return errors.New("tls: application data record requested before handshake complete" ) + return c.in.setErrorLocked(errors.New("tls: application data record requested before handshake complete") ) } } @@ -548,7 +541,7 @@ Again: // err = io.ErrUnexpectedEOF // } if e, ok := err.(net.Error); !ok || !e.Temporary() { - c. setError (err) + c. in.setErrorLocked (err) } return err } @@ -560,18 +553,18 @@ Again: // an SSLv2 client. if want == recordTypeHandshake && typ == 0x80 { c.sendAlert(alertProtocolVersion) - return errors.New("tls: unsupported SSLv2 handshake received" ) + return c.in.setErrorLocked(errors.New("tls: unsupported SSLv2 handshake received") ) } vers := uint16(b.data[1])<<8 | uint16(b.data[2]) n := int(b.data[3])<<8 | int(b.data[4]) if c.haveVers && vers != c.vers { c.sendAlert(alertProtocolVersion) - return fmt.Errorf("tls: received record with version %x when expecting version %x", vers, c.vers ) + return c.in.setErrorLocked(fmt.Errorf("tls: received record with version %x when expecting version %x", vers, c.vers) ) } if n > maxCiphertext { c.sendAlert(alertRecordOverflow) - return fmt.Errorf("tls: oversized record received with length %d", n ) + return c.in.setErrorLocked(fmt.Errorf("tls: oversized record received with length %d", n) ) } if !c.haveVers { // First message, be extra suspicious: @@ -584,7 +577,7 @@ Again: // it's probably not real. if (typ != recordTypeAlert && typ != want) || vers >= 0x1000 || n >= 0x3000 { c.sendAlert(alertUnexpectedMessage) - return fmt.Errorf("tls: first record does not look like a TLS handshake" ) + return c.in.setErrorLocked(fmt.Errorf("tls: first record does not look like a TLS handshake") ) } } if err := b.readFromUntil(c.conn, recordHeaderLen+n); err != nil { @@ -592,7 +585,7 @@ Again: err = io.ErrUnexpectedEOF } if e, ok := err.(net.Error); !ok || !e.Temporary() { - c. setError (err) + c. in.setErrorLocked (err) } return err } @@ -601,27 +594,27 @@ Again: b, c.rawInput = c.in.splitBlock(b, recordHeaderLen+n) ok, off, err := c.in.decrypt(b) if !ok { - return c.sendAlert(err ) + c.in.setErrorLocked(c.sendAlert(err) ) } b.off = off data := b.data[b.off:] if len(data) > maxPlaintext { - c.sendAlert(alertRecordOverflow) + err := c.sendAlert(alertRecordOverflow) c.in.freeBlock(b) - return c. error( ) + return c. in.setErrorLocked(err ) } switch typ { default: - c. sendAlert(alertUnexpectedMessage ) + c. in.setErrorLocked(c.sendAlert(alertUnexpectedMessage) ) case recordTypeAlert: if len(data) != 2 { - c. sendAlert(alertUnexpectedMessage ) + c. in.setErrorLocked(c.sendAlert(alertUnexpectedMessage) ) break } if alert(data[1]) == alertCloseNotify { - c. setError (io.EOF) + c. in.setErrorLocked (io.EOF) break } switch data[0] { @@ -630,24 +623,24 @@ Again: c.in.freeBlock(b) goto Again case alertLevelError: - c. setError (&net.OpError{Op: "remote error", Err: alert(data[1])}) + c. in.setErrorLocked (&net.OpError{Op: "remote error", Err: alert(data[1])}) default: - c. sendAlert(alertUnexpectedMessage ) + c. in.setErrorLocked(c.sendAlert(alertUnexpectedMessage) ) } case recordTypeChangeCipherSpec: if typ != want || len(data) != 1 || data[0] != 1 { - c. sendAlert(alertUnexpectedMessage ) + c. in.setErrorLocked(c.sendAlert(alertUnexpectedMessage) ) break } err := c.in.changeCipherSpec() if err != nil { - c. sendAlert(err.(alert )) + c. in.setErrorLocked(c.sendAlert(err.(alert) )) } case recordTypeApplicationData: if typ != want { - c. sendAlert(alertUnexpectedMessage ) + c. in.setErrorLocked(c.sendAlert(alertUnexpectedMessage) ) break } c.input = b @@ -656,7 +649,7 @@ Again: case recordTypeHandshake: // TODO(rsc): Should at least pick off connection close. if typ != want { - return c. sendAlert(alertNoRenegotiation ) + return c. in.setErrorLocked(c.sendAlert(alertNoRenegotiation) ) } c.hand.Write(data) } @@ -664,7 +657,7 @@ Again: if b != nil { c.in.freeBlock(b) } - return c. error() + return c. in.err } // sendAlert sends a TLS alert message. @@ -680,7 +673,7 @@ func (c *Conn) sendAlertLocked(err alert) error { c.writeRecord(recordTypeAlert, c.tmp[0:2]) // closeNotify is a special case in that it isn't an error: if err != alertCloseNotify { - return c. setError (&net.OpError{Op: "local error", Err: err}) + return c. out.setErrorLocked (&net.OpError{Op: "local error", Err: err}) } return nil } @@ -766,7 +759,7 @@ func (c *Conn) writeRecord(typ recordType, data []byte) (n int, err error) { c.tmp[0] = alertLevelError c.tmp[1] = byte(err.(alert)) c.writeRecord(recordTypeAlert, c.tmp[0:2]) - return n, c. setError (&net.OpError{Op: "local error", Err: err}) + return n, c. out.setErrorLocked (&net.OpError{Op: "local error", Err: err}) } } return @@ -777,7 +770,7 @@ func (c *Conn) writeRecord(typ recordType, data []byte) (n int, err error) { // c.in.Mutex < L; c.out.Mutex < L. func (c *Conn) readHandshake() (interface{}, error) { for c.hand.Len() < 4 { - if err := c. error() ; err != nil { + if err := c. in.err ; err != nil { return nil, err } if err := c.readRecord(recordTypeHandshake); err != nil { @@ -788,11 +781,10 @@ func (c *Conn) readHandshake() (interface{}, error) { data := c.hand.Bytes() n := int(data[1])<<16 | int(data[2])<<8 | int(data[3]) if n > maxHandshake { - c.sendAlert(alertInternalError) - return nil, c.error() + return nil, c.in.setErrorLocked(c.sendAlert(alertInternalError)) } for c.hand.Len() < 4+n { - if err := c. error() ; err != nil { + if err := c. in.err ; err != nil { return nil, err } if err := c.readRecord(recordTypeHandshake); err != nil { @@ -831,8 +823,7 @@ func (c *Conn) readHandshake() (interface{}, error) { case typeFinished: m = new(finishedMsg) default: - c.sendAlert(alertUnexpectedMessage) - return nil, alertUnexpectedMessage + return nil, c.in.setErrorLocked(c.sendAlert(alertUnexpectedMessage)) } // The handshake message unmarshallers @@ -841,25 +832,24 @@ func (c *Conn) readHandshake() (interface{}, error) { data = append([]byte(nil), data...) if !m.unmarshal(data) { - c.sendAlert(alertUnexpectedMessage) - return nil, alertUnexpectedMessage + return nil, c.in.setErrorLocked(c.sendAlert(alertUnexpectedMessage)) } return m, nil } // Write writes data to the connection. func (c *Conn) Write(b []byte) (int, error) { - if err := c.error(); err != nil { - return 0, err - } - if err := c.Handshake(); err != nil { - return 0, c.setError(err) + return 0, err } c.out.Lock() defer c.out.Unlock() + if err := c.out.err; err != nil { + return 0, err + } + if !c.handshakeComplete { return 0, alertInternalError } @@ -878,14 +868,14 @@ func (c *Conn) Write(b []byte) (int, error) { if _, ok := c.out.cipher.(cipher.BlockMode); ok { n, err := c.writeRecord(recordTypeApplicationData, b[:1]) if err != nil { - return n, c. setError (err) + return n, c. out.setErrorLocked (err) } m, b = 1, b[1:] } } n, err := c.writeRecord(recordTypeApplicationData, b) - return n + m, c. setError (err) + return n + m, c. out.setErrorLocked (err) } // Read can be made to time out and return a net.Error with Timeout() == true @@ -894,6 +884,11 @@ func (c *Conn) Read(b []byte) (n int, err error) { if err = c.Handshake(); err != nil { return } + if len(b) == 0 { + // Put this after Handshake, in case people were calling + // Read(nil) for the side effect of the Handshake. + return + } c.in.Lock() defer c.in.Unlock() @@ -902,13 +897,13 @@ func (c *Conn) Read(b []byte) (n int, err error) { // CBC IV. So this loop ignores a limited number of empty records. const maxConsecutiveEmptyRecords = 100 for emptyRecordCount := 0; emptyRecordCount <= maxConsecutiveEmptyRecords; emptyRecordCount++ { - for c.input == nil && c. error() == nil { + for c.input == nil && c. in.err == nil { if err := c.readRecord(recordTypeApplicationData); err != nil { // Soft error, like EAGAIN return 0, err } } - if err := c. error() ; err != nil { + if err := c. in.err ; err != nil { return 0, err } @@ -918,6 +913,25 @@ func (c *Conn) Read(b []byte) (n int, err error) { c.input = nil } + // If a close-notify alert is waiting, read it so that + // we can return (n, EOF) instead of (n, nil), to signal + // to the HTTP response reading goroutine that the + // connection is now closed. This eliminates a race + // where the HTTP response reading goroutine would + // otherwise not observe the EOF until its next read, + // by which time a client goroutine might have already + // tried to reuse the HTTP connection for a new + // request. + // See https://codereview.appspot.com/76400046 + // and http://golang.org/issue/3514 + if ri := c.rawInput; ri != nil && + n != 0 && err == nil && + c.input == nil && len(ri.data) > 0 && recordType(ri.data[0]) == recordTypeAlert { + if recErr := c.readRecord(recordTypeApplicationData); recErr != nil { + err = recErr // will be io.EOF on closeNotify + } + } + if n != 0 || err != nil { return n, err } @@ -949,16 +963,19 @@ func (c *Conn) Close() error { func (c *Conn) Handshake() error { c.handshakeMutex.Lock() defer c.handshakeMutex.Unlock() - if err := c. error() ; err != nil { + if err := c. handshakeErr ; err != nil { return err } if c.handshakeComplete { return nil } + if c.isClient { - return c.clientHandshake() + c.handshakeErr = c.clientHandshake() + } else { + c.handshakeErr = c.serverHandshake() } - return c. serverHandshake() + return c. handshakeErr } // ConnectionState returns basic TLS details about the connection. @@ -969,6 +986,7 @@ func (c *Conn) ConnectionState() ConnectionState { var state ConnectionState state.HandshakeComplete = c.handshakeComplete if c.handshakeComplete { + state.Version = c.vers state.NegotiatedProtocol = c.clientProtocol state.DidResume = c.didResume state.NegotiatedProtocolIsMutual = !c.clientProtocolFallback

b/libgo/go/crypto/tls/generate_cert.go diff --git a/libgo/go/crypto/tls/generate_cert.go (file)

index 1b4830c .. 5c6d839 100644 --- a/ libgo/go/crypto/tls/generate_cert.go +++ b/ libgo/go/crypto/tls/generate_cert.go @@ -58,12 +58,6 @@ func main() { notAfter := notBefore.Add(*validFor) - // end of ASN.1 time - endOfTime := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC) - if notAfter.After(endOfTime) { - notAfter = endOfTime - } - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) if err != nil {

b/libgo/go/crypto/tls/handshake_client.go diff --git a/libgo/go/crypto/tls/handshake_client.go (file)

index fd1303e .. a320fde 100644 --- a/ libgo/go/crypto/tls/handshake_client.go +++ b/ libgo/go/crypto/tls/handshake_client.go @@ -33,13 +33,17 @@ func (c *Conn) clientHandshake() error { c.config = defaultConfig() } + if len(c.config.ServerName) == 0 && !c.config.InsecureSkipVerify { + return errors.New("tls: either ServerName or InsecureSkipVerify must be specified in the tls.Config") + } + hello := &clientHelloMsg{ vers: c.config.maxVersion(), compressionMethods: []uint8{compressionNone}, random: make([]byte, 32), ocspStapling: true, serverName: c.config.ServerName, - supportedCurves: []uint16{curveP256, curveP384, curveP521} , + supportedCurves: c.config.curvePreferences() , supportedPoints: []uint8{pointFormatUncompressed}, nextProtoNeg: len(c.config.NextProtos) > 0, secureRenegotiation: true, @@ -497,7 +501,7 @@ func (hs *clientHandshakeState) readFinished() error { c := hs.c c.readRecord(recordTypeChangeCipherSpec) - if err := c.error(); err != nil { + if err := c. in. error(); err != nil { return err }

b/libgo/go/crypto/tls/handshake_messages.go diff --git a/libgo/go/crypto/tls/handshake_messages.go (file)

index fbdd0b9 .. 7bcaa5e 100644 --- a/ libgo/go/crypto/tls/handshake_messages.go +++ b/ libgo/go/crypto/tls/handshake_messages.go @@ -16,7 +16,7 @@ type clientHelloMsg struct { nextProtoNeg bool serverName string ocspStapling bool - supportedCurves [] uint16 + supportedCurves [] CurveID supportedPoints []uint8 ticketSupported bool sessionTicket []uint8 @@ -39,7 +39,7 @@ func (m *clientHelloMsg) equal(i interface{}) bool { m.nextProtoNeg == m1.nextProtoNeg && m.serverName == m1.serverName && m.ocspStapling == m1.ocspStapling && - eq Uint16 s(m.supportedCurves, m1.supportedCurves) && + eq CurveID s(m.supportedCurves, m1.supportedCurves) && bytes.Equal(m.supportedPoints, m1.supportedPoints) && m.ticketSupported == m1.ticketSupported && bytes.Equal(m.sessionTicket, m1.sessionTicket) && @@ -357,10 +357,10 @@ func (m *clientHelloMsg) unmarshal(data []byte) bool { return false } numCurves := l / 2 - m.supportedCurves = make([] uint16 , numCurves) + m.supportedCurves = make([] CurveID , numCurves) d := data[2:] for i := 0; i < numCurves; i++ { - m.supportedCurves[i] = uint16(d[0])<<8 | uint16 (d[1]) + m.supportedCurves[i] = CurveID(d[0])<<8 | CurveID (d[1]) d = d[2:] } case extensionSupportedPoints: @@ -1294,6 +1294,18 @@ func eqUint16s(x, y []uint16) bool { return true } +func eqCurveIDs(x, y []CurveID) bool { + if len(x) != len(y) { + return false + } + for i, v := range x { + if y[i] != v { + return false + } + } + return true +} + func eqStrings(x, y []string) bool { if len(x) != len(y) { return false

b/libgo/go/crypto/tls/handshake_messages_test.go diff --git a/libgo/go/crypto/tls/handshake_messages_test.go (file)

index 4f569ee .. f46aabd 100644 --- a/ libgo/go/crypto/tls/handshake_messages_test.go +++ b/ libgo/go/crypto/tls/handshake_messages_test.go @@ -125,9 +125,9 @@ func (*clientHelloMsg) Generate(rand *rand.Rand, size int) reflect.Value { } m.ocspStapling = rand.Intn(10) > 5 m.supportedPoints = randomBytes(rand.Intn(5)+1, rand) - m.supportedCurves = make([] uint16 , rand.Intn(5)+1) + m.supportedCurves = make([] CurveID , rand.Intn(5)+1) for i := range m.supportedCurves { - m.supportedCurves[i] = uint16 (rand.Intn(30000)) + m.supportedCurves[i] = CurveID (rand.Intn(30000)) } if rand.Intn(10) > 5 { m.ticketSupported = true

b/libgo/go/crypto/tls/handshake_server.go diff --git a/libgo/go/crypto/tls/handshake_server.go (file)

index 12e5ff1 .. 75111eb 100644 --- a/ libgo/go/crypto/tls/handshake_server.go +++ b/ libgo/go/crypto/tls/handshake_server.go @@ -117,12 +117,14 @@ func (hs *serverHandshakeState) readClientHello() (isResume bool, err error) { hs.hello = new(serverHelloMsg) supportedCurve := false + preferredCurves := config.curvePreferences() Curves: for _, curve := range hs.clientHello.supportedCurves { - switch curve { - case curveP256, curveP384, curveP521: - supportedCurve = true - break Curves + for _, supported := range preferredCurves { + if supported == curve { + supportedCurve = true + break Curves + } } } @@ -468,7 +470,7 @@ func (hs *serverHandshakeState) readFinished() error { c := hs.c c.readRecord(recordTypeChangeCipherSpec) - if err := c.error(); err != nil { + if err := c. in. error(); err != nil { return err }

b/libgo/go/crypto/tls/handshake_server_test.go diff --git a/libgo/go/crypto/tls/handshake_server_test.go (file)

index 4f41ab9 .. c3e3678 100644 --- a/ libgo/go/crypto/tls/handshake_server_test.go +++ b/ libgo/go/crypto/tls/handshake_server_test.go @@ -121,7 +121,7 @@ func TestTLS12OnlyCipherSuites(t *testing.T) { TLS_RSA_WITH_RC4_128_SHA, }, compressionMethods: []uint8{compressionNone}, - supportedCurves: [] uint16{curveP256, curveP384, c urveP521}, + supportedCurves: [] CurveID{CurveP256, CurveP384, C urveP521}, supportedPoints: []uint8{pointFormatUncompressed}, } @@ -195,6 +195,23 @@ func testHandshake(clientConfig, serverConfig *Config) (state ConnectionState, e return } +func TestVersion(t *testing.T) { + serverConfig := &Config{ + Certificates: testConfig.Certificates, + MaxVersion: VersionTLS11, + } + clientConfig := &Config{ + InsecureSkipVerify: true, + } + state, err := testHandshake(clientConfig, serverConfig) + if err != nil { + t.Fatalf("handshake failed: %s", err) + } + if state.Version != VersionTLS11 { + t.Fatalf("Incorrect version %x, should be %x", state.Version, VersionTLS11) + } +} + func TestCipherSuitePreference(t *testing.T) { serverConfig := &Config{ CipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA, TLS_RSA_WITH_AES_128_CBC_SHA, TLS_ECDHE_RSA_WITH_RC4_128_SHA},

b/libgo/go/crypto/tls/key_agreement.go diff --git a/libgo/go/crypto/tls/key_agreement.go (file)

index 861faf0 .. f38b701 100644 --- a/ libgo/go/crypto/tls/key_agreement.go +++ b/ libgo/go/crypto/tls/key_agreement.go @@ -141,7 +141,7 @@ func hashForServerKeyExchange(sigType, hashFunc uint8, version uint16, slices .. // pickTLS12HashForSignature returns a TLS 1.2 hash identifier for signing a // ServerKeyExchange given the signature type being used and the client's -// adverti z ed list of supported signature and hash combinations. +// adverti s ed list of supported signature and hash combinations. func pickTLS12HashForSignature(sigType uint8, clientSignatureAndHashes []signatureAndHash) (uint8, error) { if len(clientSignatureAndHashes) == 0 { // If the client didn't specify any signature_algorithms @@ -163,6 +163,20 @@ func pickTLS12HashForSignature(sigType uint8, clientSignatureAndHashes []signatu return 0, errors.New("tls: client doesn't support any common hash functions") } +func curveForCurveID(id CurveID) (elliptic.Curve, bool) { + switch id { + case CurveP256: + return elliptic.P256(), true + case CurveP384: + return elliptic.P384(), true + case CurveP521: + return elliptic.P521(), true + default: + return nil, false + } + +} + // ecdheRSAKeyAgreement implements a TLS key agreement where the server // generates a ephemeral EC public/private key pair and signs it. The // pre-master secret is then calculated using ECDH. The signature may @@ -176,23 +190,16 @@ type ecdheKeyAgreement struct { } func (ka *ecdheKeyAgreement) generateServerKeyExchange(config *Config, cert *Certificate, clientHello *clientHelloMsg, hello *serverHelloMsg) (*serverKeyExchangeMsg, error) { - var curveid uint16 - -Curve: - for _