Merge pull request #2081 from blahgeek/support-index-v3

Preliminary support for index format v3

Author: Byron

git/index/fun.py

@@ -36,7 +36,7 @@
 )
 from git.util import IndexFileSHA1Writer, finalize_process
 
-from .typ import BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT
+from .typ import CE_EXTENDED, BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT
 from .util import pack, unpack
 
 # typing -----------------------------------------------------------------------------
@@ -158,7 +158,7 @@ def write_cache(
     write = stream_sha.write
 
     # Header
-    version = 2
+    version = 3 if any(entry.extended_flags for entry in entries) else 2
     write(b"DIRC")
     write(pack(">LL", version, len(entries)))
 
@@ -172,6 +172,8 @@ def write_cache(
         plen = len(path) & CE_NAMEMASK  # Path length
         assert plen == len(path), "Path %s too long to fit into index" % entry.path
         flags = plen | (entry.flags & CE_NAMEMASK_INV)  # Clear possible previous values.
+        if entry.extended_flags:
+            flags |= CE_EXTENDED
         write(
             pack(
                 ">LLLLLL20sH",
@@ -185,6 +187,8 @@ def write_cache(
                 flags,
             )
         )
+        if entry.extended_flags:
+            write(pack(">H", entry.extended_flags))
         write(path)
         real_size = (tell() - beginoffset + 8) & ~7
         write(b"\0" * ((beginoffset + real_size) - tell()))
@@ -206,8 +210,7 @@ def read_header(stream: IO[bytes]) -> Tuple[int, int]:
     unpacked = cast(Tuple[int, int], unpack(">LL", stream.read(4 * 2)))
     version, num_entries = unpacked
 
-    # TODO: Handle version 3: extended data, see read-cache.c.
-    assert version in (1, 2), "Unsupported git index version %i, only 1 and 2 are supported" % version
+    assert version in (1, 2, 3), "Unsupported git index version %i, only 1, 2, and 3 are supported" % version
     return version, num_entries
 
 
@@ -260,12 +263,15 @@ def read_cache(
         ctime = unpack(">8s", read(8))[0]
         mtime = unpack(">8s", read(8))[0]
         (dev, ino, mode, uid, gid, size, sha, flags) = unpack(">LLLLLL20sH", read(20 + 4 * 6 + 2))
+        extended_flags = 0
+        if flags & CE_EXTENDED:
+            extended_flags = unpack(">H", read(2))[0]
         path_size = flags & CE_NAMEMASK
         path = read(path_size).decode(defenc)
 
         real_size = (tell() - beginoffset + 8) & ~7
         read((beginoffset + real_size) - tell())
-        entry = IndexEntry((mode, sha, flags, path, ctime, mtime, dev, ino, uid, gid, size))
+        entry = IndexEntry((mode, sha, flags, path, ctime, mtime, dev, ino, uid, gid, size, extended_flags))
         # entry_key would be the method to use, but we save the effort.
         entries[(path, entry.stage)] = entry
         count += 1

git/index/typ.py

@@ -32,6 +32,9 @@
 CE_VALID = 0x8000
 CE_STAGESHIFT = 12
 
+CE_EXT_SKIP_WORKTREE = 0x4000
+CE_EXT_INTENT_TO_ADD = 0x2000
+
 # } END invariants
 
 
@@ -87,6 +90,8 @@ class BaseIndexEntryHelper(NamedTuple):
     uid: int = 0
     gid: int = 0
     size: int = 0
+    # version 3 extended flags, only when (flags & CE_EXTENDED) is set
+    extended_flags: int = 0
 
 
 class BaseIndexEntry(BaseIndexEntryHelper):
@@ -102,7 +107,7 @@ def __new__(
         cls,
         inp_tuple: Union[
             Tuple[int, bytes, int, PathLike],
-            Tuple[int, bytes, int, PathLike, bytes, bytes, int, int, int, int, int],
+            Tuple[int, bytes, int, PathLike, bytes, bytes, int, int, int, int, int, int],
         ],
     ) -> "BaseIndexEntry":
         """Override ``__new__`` to allow construction from a tuple for backwards
@@ -134,6 +139,14 @@ def stage(self) -> int:
         """
         return (self.flags & CE_STAGEMASK) >> CE_STAGESHIFT
 
+    @property
+    def skip_worktree(self) -> bool:
+        return (self.extended_flags & CE_EXT_SKIP_WORKTREE) > 0
+
+    @property
+    def intent_to_add(self) -> bool:
+        return (self.extended_flags & CE_EXT_INTENT_TO_ADD) > 0
+
     @classmethod
     def from_blob(cls, blob: Blob, stage: int = 0) -> "BaseIndexEntry":
         """:return: Fully equipped BaseIndexEntry at the given stage"""

test/fixtures/index_extended_flags

@@ -1218,6 +1218,48 @@ def test_index_add_non_normalized_path(self, rw_repo):
 
         rw_repo.index.add(non_normalized_path)
 
+    def test_index_file_v3(self):
+        index = IndexFile(self.rorepo, fixture_path("index_extended_flags"))
+        assert index.entries
+        assert index.version == 3
+        assert len(index.entries) == 4
+        assert index.entries[("init.t", 0)].skip_worktree
+
+        # Write the data - it must match the original.
+        with tempfile.NamedTemporaryFile() as tmpfile:
+            index.write(tmpfile.name)
+            assert Path(tmpfile.name).read_bytes() == Path(fixture_path("index_extended_flags")).read_bytes()
+
+    @with_rw_directory
+    def test_index_file_v3_with_git_command(self, tmp_dir):
+        tmp_dir = Path(tmp_dir)
+        with cwd(tmp_dir):
+            git = Git(tmp_dir)
+            git.init()
+
+            file = tmp_dir / "file.txt"
+            file.write_text("hello")
+            git.add("--intent-to-add", "file.txt")  # intent-to-add sets extended flag
+
+            repo = Repo(tmp_dir)
+            index = repo.index
+
+            assert len(index.entries) == 1
+            assert index.version == 3
+            entry = list(index.entries.values())[0]
+            assert entry.path == "file.txt"
+            assert entry.intent_to_add
+
+            file2 = tmp_dir / "file2.txt"
+            file2.write_text("world")
+            index.add(["file2.txt"])
+            index.write()
+
+            status_str = git.status(porcelain=True)
+            status_lines = status_str.splitlines()
+            assert " A file.txt" in status_lines
+            assert "A  file2.txt" in status_lines
+
 
 class TestIndexUtils:
     @pytest.mark.parametrize("file_path_type", [str, Path])