Merge pull request #37 from scjalliance/objfile-format

Added support for objfile format

Author: mcuadros

commit.go

@@ -61,7 +61,7 @@ func (c *Commit) Decode(o core.Object) (err error) {
 	if err != nil {
 		return err
 	}
-	defer close(reader, &err)
+	defer checkClose(reader, &err)
 
 	r := bufio.NewReader(reader)
 

common.go

@@ -22,21 +22,21 @@ func countLines(s string) int {
 	return nEOL + 1
 }
 
-// close is used with defer to close the given io.Closer and check its
+// checkClose is used with defer to close the given io.Closer and check its
 // returned error value. If Close returns an error and the given *error
 // is not nil, *error is set to the error returned by Close.
 //
-// close is typically used with named return values like so:
+// checkClose is typically used with named return values like so:
 //
 //   func do(obj *Object) (err error) {
 //     w, err := obj.Writer()
 //     if err != nil {
 //       return nil
 //     }
-//     defer close(w, &err)
+//     defer checkClose(w, &err)
 //     // work with w
 //   }
-func close(c io.Closer, err *error) {
+func checkClose(c io.Closer, err *error) {
 	if cerr := c.Close(); cerr != nil && *err == nil {
 		*err = cerr
 	}

core/object.go

@@ -8,6 +8,8 @@ import (
 
 var (
 	ObjectNotFoundErr = errors.New("object not found")
+	// ErrInvalidType is returned when an invalid object type is provided.
+	ErrInvalidType = errors.New("invalid object type")
 )
 
 // TODO: Consider adding a Hash function to the ObjectReader and ObjectWriter
@@ -79,6 +81,11 @@ func (t ObjectType) Bytes() []byte {
 	return []byte(t.String())
 }
 
+// Valid returns true if t is a valid ObjectType.
+func (t ObjectType) Valid() bool {
+	return t >= CommitObject && t <= REFDeltaObject
+}
+
 // ParseObjectType parses a string representation of ObjectType. It returns an
 // error on parse failure.
 func ParseObjectType(value string) (typ ObjectType, err error) {
@@ -96,7 +103,7 @@ func ParseObjectType(value string) (typ ObjectType, err error) {
 	case "ref-delta":
 		typ = REFDeltaObject
 	default:
-		err = errors.New("unable to parse object type")
+		err = ErrInvalidType
 	}
 	return
 }

file.go

@@ -23,7 +23,7 @@ func (f *File) Contents() (content string, err error) {
 	if err != nil {
 		return "", err
 	}
-	defer close(reader, &err)
+	defer checkClose(reader, &err)
 
 	buf := new(bytes.Buffer)
 	buf.ReadFrom(reader)

formats/objfile/common.go

@@ -0,0 +1,79 @@
+package objfile
+
+import (
+	"errors"
+	"io"
+	"strconv"
+
+	"gopkg.in/src-d/go-git.v3/core"
+)
+
+var (
+	// ErrClosed is returned when the objfile Reader or Writer is already closed.
+	ErrClosed = errors.New("objfile: already closed")
+	// ErrHeader is returned when the objfile has an invalid header.
+	ErrHeader = errors.New("objfile: invalid header")
+	// ErrNegativeSize is returned when a negative object size is declared.
+	ErrNegativeSize = errors.New("objfile: negative object size")
+)
+
+type header struct {
+	t    core.ObjectType
+	size int64
+}
+
+func (h *header) Read(r io.Reader) error {
+	t, err := h.readSlice(r, ' ')
+	if err != nil {
+		return err
+	}
+
+	h.t, err = core.ParseObjectType(string(t))
+	if err != nil {
+		return err
+	}
+
+	size, err := h.readSlice(r, 0)
+	if err != nil {
+		return err
+	}
+
+	h.size, err = strconv.ParseInt(string(size), 10, 64)
+	if err != nil {
+		return ErrHeader
+	}
+
+	if h.size < 0 {
+		return ErrNegativeSize
+	}
+
+	return nil
+}
+
+func (h *header) Write(w io.Writer) error {
+	b := h.t.Bytes()
+	b = append(b, ' ')
+	b = append(b, []byte(strconv.FormatInt(h.size, 10))...)
+	b = append(b, 0)
+	_, err := w.Write(b)
+	return err
+}
+
+// readSlice reads one byte at a time from r until it encounters delim or an
+// error.
+func (h *header) readSlice(r io.Reader, delim byte) ([]byte, error) {
+	var buf [1]byte
+	value := make([]byte, 0, 16)
+	for {
+		if n, err := r.Read(buf[:]); err != nil && (err != io.EOF || n == 0) {
+			if err == io.EOF {
+				return nil, ErrHeader
+			}
+			return nil, err
+		}
+		if buf[0] == delim {
+			return value, nil
+		}
+		value = append(value, buf[0])
+	}
+}

formats/objfile/common_test.go

@@ -0,0 +1,100 @@
+package objfile
+
+import (
+	"bytes"
+	"encoding/base64"
+	"testing"
+
+	. "gopkg.in/check.v1"
+	"gopkg.in/src-d/go-git.v3/core"
+)
+
+type objfileFixture struct {
+	hash    string          // hash of data
+	t       core.ObjectType // object type
+	content string          // base64-encoded content
+	data    string          // base64-encoded objfile data
+}
+
+var objfileFixtures = []objfileFixture{
+	{
+		"e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
+		core.BlobObject,
+		base64.StdEncoding.EncodeToString([]byte("")),
+		"eAFLyslPUjBgAAAJsAHw",
+	},
+	{
+		"a8a940627d132695a9769df883f85992f0ff4a43",
+		core.BlobObject,
+		base64.StdEncoding.EncodeToString([]byte("this is a test")),
+		"eAFLyslPUjA0YSjJyCxWAKJEhZLU4hIAUDYHOg==",
+	},
+	{
+		"4dc2174801ac4a3d36886210fd086fbe134cf7b2",
+		core.BlobObject,
+		base64.StdEncoding.EncodeToString([]byte("this\nis\n\n\na\nmultiline\n\ntest.\n")),
+		"eAFLyslPUjCyZCjJyCzmAiIurkSu3NKcksyczLxULq6S1OISPS4A1I8LMQ==",
+	},
+	{
+		"13e6f47dd57798bfdc728d91f5c6d7f40c5bb5fc",
+		core.BlobObject,
+		base64.StdEncoding.EncodeToString([]byte("this tests\r\nCRLF\r\nencoded files.\r\n")),
+		"eAFLyslPUjA2YSjJyCxWKEktLinm5XIO8nHj5UrNS85PSU1RSMvMSS3W4+UCABp3DNE=",
+	},
+	{
+		"72a7bc4667ab068e954172437b993d9fbaa137cb",
+		core.BlobObject,
+		base64.StdEncoding.EncodeToString([]byte("[email protected]")),
+		"eAFLyslPUjA0YyhJLS5xSK1IzC3ISdVLzs8FAGVtCIA=",
+	},
+	{
+		"bb2b40e85ec0455d1de72daff71583f0dd72a33f",
+		core.BlobObject,
+		base64.StdEncoding.EncodeToString([]byte("package main\r\n\r\nimport (\r\n\t\"fmt\"\r\n\t\"io\"\r\n\t\"os\"\r\n\r\n\t\"gopkg.in/src-d/go-git.v3\"\r\n)\r\n\r\nfunc main() {\r\n\tfmt.Printf(\"Retrieving %q ...\\n\", os.Args[2])\r\n\tr, err := git.NewRepository(os.Args[2], nil)\r\n\tif err != nil {\r\n\t\tpanic(err)\r\n\t}\r\n\r\n\tif err := r.Pull(\"origin\", \"refs/heads/master\"); err != nil {\r\n\t\tpanic(err)\r\n\t}\r\n\r\n\tdumpCommits(r)\r\n}\r\n\r\nfunc dumpCommits(r *git.Repository) {\r\n\titer := r.Commits()\r\n\tdefer iter.Close()\r\n\r\n\tfor {\r\n\t\tcommit, err := iter.Next()\r\n\t\tif err != nil {\r\n\t\t\tif err == io.EOF {\r\n\t\t\t\tbreak\r\n\t\t\t}\r\n\r\n\t\t\tpanic(err)\r\n\t\t}\r\n\r\n\t\tfmt.Println(commit)\r\n\t}\r\n}\r\n")),
+		"eAGNUU1LAzEU9JpC/0NcEFJps2ARQdmDFD3W0qt6SHez8dHdZH1JqyL+d/Oy/aDgQVh47LzJTGayatyKX99MzzpVrpXRvFVgh4PhANrOYeBiOGBZ3YaMJrg0nI+D/o3r1kaCzT2Wkyo3bmIgyO00rkfEqDe2TIJixL/jgagjFwg21CJb6oCgt2ANv3jnUsoXm4258/IejX++eo0CDMdcI/LbgpPuXH8sdec8BIdf4sgccwsN0aFO9POCgGTIOmWhFFGE9j/p1jtWFEW52DSNyByCAXLPUNc+f9Oq8nmrfNCYje7+o1lt2m7m2haCF2SVnFL6kw2/pBzHEH0rEH0oI8q9BF220nWEaSdnjfNaRDDCtcM+WZnsDgUl4lx/BuKxv6rYY0XBwcmHp8deh7EVarWmQ7uC2Glre/TweI0VvTk5xaTx+wWX66Gs",
+	},
+	{
+		"e94db0f9ffca44dc7bade6a3591f544183395a7c",
+		core.TreeObject,
+		"MTAwNjQ0IFRlc3QgMS50eHQAqKlAYn0TJpWpdp34g/hZkvD/SkMxMDA2NDQgVGVzdCAyLnR4dABNwhdIAaxKPTaIYhD9CG++E0z3sjEwMDY0NCBUZXN0IDMudHh0ABPm9H3Vd5i/3HKNkfXG1/QMW7X8MTAwNjQ0IFRlc3QgNC50eHQAcqe8RmerBo6VQXJDe5k9n7qhN8sxMDA2NDQgVGVzdCA1LnR4dAC7K0DoXsBFXR3nLa/3FYPw3XKjPw==",
+		"eAErKUpNVTC0NGAwNDAwMzFRCEktLlEw1CupKGFYsdIhqVZYberKsrk/mn9ETvrw38sZWZURWJXvIXEPxjVetmYdSQJ/OfL3Cft834SsyhisSvjZl9qr5TP23ynqnfj12PUvPNFb/yCrMgGrKlq+xy19NVvfVMci5+qZtvN3LTQ/jazKFKxqt7bDi7gDrrGyz3XXfxdt/nC3aLE9AA2STmk=",
+	},
+	{
+		"9d7f8a56eaf92469dee8a856e716a03387ddb076",
+		core.CommitObject,
+		"dHJlZSBlOTRkYjBmOWZmY2E0NGRjN2JhZGU2YTM1OTFmNTQ0MTgzMzk1YTdjCmF1dGhvciBKb3NodWEgU2pvZGluZyA8am9zaHVhLnNqb2RpbmdAc2NqYWxsaWFuY2UuY29tPiAxNDU2NTMxNTgzIC0wODAwCmNvbW1pdHRlciBKb3NodWEgU2pvZGluZyA8am9zaHVhLnNqb2RpbmdAc2NqYWxsaWFuY2UuY29tPiAxNDU2NTMxNTgzIC0wODAwCgpUZXN0IENvbW1pdAo=",
+		"eAGtjksOgjAUAF33FO8CktZ+aBNjTNy51Qs8Xl8FAjSh5f4SvILLmcVkKM/zUOEi3amuzMDBxE6mkBKhMZHaDiM71DaoZI1RXutgsSWBW+3zCs9c+g3hNeY4LB+4jgc35cf3QiNO04ALcUN5voEy1lmtrNdwll5Ksdt9oPIfUuLNpcLjCIov3ApFmQ==",
+	},
+}
+
+func Test(t *testing.T) { TestingT(t) }
+
+type SuiteCommon struct{}
+
+var _ = Suite(&SuiteCommon{})
+
+func (s *SuiteCommon) TestHeaderReadEmpty(c *C) {
+	var h header
+	c.Assert(h.Read(new(bytes.Buffer)), Equals, ErrHeader)
+}
+
+func (s *SuiteCommon) TestHeaderReadGarbage(c *C) {
+	var h header
+	c.Assert(h.Read(bytes.NewBuffer([]byte{1, 2, 3, 4, 5})), Equals, ErrHeader)
+	c.Assert(h.Read(bytes.NewBuffer([]byte{1, 2, 3, 4, 5, '0'})), Equals, ErrHeader)
+}
+
+func (s *SuiteCommon) TestHeaderReadInvalidType(c *C) {
+	var h header
+	c.Assert(h.Read(bytes.NewBuffer([]byte{1, 2, ' ', 4, 5, 0})), Equals, core.ErrInvalidType)
+}
+
+func (s *SuiteCommon) TestHeaderReadInvalidSize(c *C) {
+	var h header
+	c.Assert(h.Read(bytes.NewBuffer([]byte{'b', 'l', 'o', 'b', ' ', 'a', 0})), Equals, ErrHeader)
+}
+
+func (s *SuiteCommon) TestHeaderReadNegativeSize(c *C) {
+	var h header
+	c.Assert(h.Read(bytes.NewBuffer([]byte{'b', 'l', 'o', 'b', ' ', '-', '1', 0})), Equals, ErrNegativeSize)
+}

formats/objfile/reader.go

@@ -0,0 +1,123 @@
+package objfile
+
+import (
+	"errors"
+	"io"
+
+	"gopkg.in/src-d/go-git.v3/core"
+
+	"github.com/klauspost/compress/zlib"
+)
+
+var (
+	// ErrZLib is returned when the objfile contains invalid zlib data.
+	ErrZLib = errors.New("objfile: invalid zlib data")
+)
+
+// Reader reads and decodes compressed objfile data from a provided io.Reader.
+//
+// Reader implements io.ReadCloser. Close should be called when finished with
+// the Reader. Close will not close the underlying io.Reader.
+type Reader struct {
+	header header
+	hash   core.Hash // final computed hash stored after Close
+
+	r            io.Reader     // provided reader wrapped in decompressor and tee
+	decompressor io.ReadCloser // provided reader wrapped in decompressor, retained for calling Close
+	h            core.Hasher   // streaming SHA1 hash of decoded data
+}
+
+// NewReader returns a new Reader reading from r.
+//
+// Calling NewReader causes it to immediately read in header data from r
+// containing size and type information. Any errors encountered in that
+// process will be returned in err.
+//
+// The returned Reader implements io.ReadCloser. Close should be called when
+// finished with the Reader. Close will not close the underlying io.Reader.
+func NewReader(r io.Reader) (*Reader, error) {
+	reader := &Reader{}
+	return reader, reader.init(r)
+}
+
+// init prepares the zlib decompressor for the given input as well as a hasher
+// for computing its hash.
+//
+// init immediately reads header data from the input and stores it. This leaves
+// the Reader in a state that is ready to read content.
+func (r *Reader) init(input io.Reader) (err error) {
+	r.decompressor, err = zlib.NewReader(input)
+	if err != nil {
+		// TODO: Make this error match the ZLibErr in formats/packfile/reader.go?
+		return ErrZLib
+	}
+
+	err = r.header.Read(r.decompressor)
+	if err != nil {
+		r.decompressor.Close()
+		return
+	}
+
+	r.h = core.NewHasher(r.header.t, r.header.size)
+	r.r = io.TeeReader(r.decompressor, r.h) // All reads from the decompressor also write to the hash
+
+	return
+}
+
+// Read reads len(p) bytes into p from the object data stream. It returns
+// the number of bytes read (0 <= n <= len(p)) and any error encountered. Even
+// if Read returns n < len(p), it may use all of p as scratch space during the
+// call.
+//
+// If Read encounters the end of the data stream it will return err == io.EOF,
+// either in the current call if n > 0 or in a subsequent call.
+func (r *Reader) Read(p []byte) (n int, err error) {
+	if r.r == nil {
+		return 0, ErrClosed
+	}
+
+	return r.r.Read(p)
+}
+
+// Type returns the type of the object.
+func (r *Reader) Type() core.ObjectType {
+	return r.header.t
+}
+
+// Size returns the uncompressed size of the object in bytes.
+func (r *Reader) Size() int64 {
+	return r.header.size
+}
+
+// Hash returns the hash of the object data stream that has been read so far.
+// It can be called before or after Close.
+func (r *Reader) Hash() core.Hash {
+	if r.r != nil {
+		return r.h.Sum() // Not yet closed, return hash of data read so far
+	}
+	return r.hash
+}
+
+// Close releases any resources consumed by the Reader.
+//
+// Calling Close does not close the wrapped io.Reader originally passed to
+// NewReader.
+func (r *Reader) Close() (err error) {
+	if r.r == nil {
+		// TODO: Consider returning ErrClosed here?
+		return nil // Already closed
+	}
+
+	// Release the decompressor's resources
+	err = r.decompressor.Close()
+
+	// Save the hash because we're about to throw away the hasher
+	r.hash = r.h.Sum()
+
+	// Release references
+	r.r = nil // Indicates closed state
+	r.decompressor = nil
+	r.h.Hash = nil
+
+	return
+}